{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fine-Tuning a BERT Model and Create a Text Classifier\n",
    "\n",
    "In the previous section, we've already performed the Feature Engineering to create BERT embeddings from the `reviews_body` text using the pre-trained BERT model, and split the dataset into train, validation and test files. To optimize for Tensorflow training, we saved the files in TFRecord format. \n",
    "\n",
    "Now, let’s fine-tune the BERT model to our Customer Reviews Dataset and add a new classification layer to predict the `star_rating` for a given `review_body`.\n",
    "\n",
    "![BERT Training](img/bert_training.png)\n",
    "\n",
    "As mentioned earlier, BERT’s attention mechanism is called a Transformer. This is, not coincidentally, the name of the popular BERT Python library, “Transformers,” maintained by a company called HuggingFace. \n",
    "\n",
    "We will use a variant of BERT called [**DistilBert**](https://arxiv.org/pdf/1910.01108.pdf) which requires less memory and compute, but maintains very good accuracy on our dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import random\n",
    "import pandas as pd\n",
    "from glob import glob\n",
    "import argparse\n",
    "import json\n",
    "import subprocess\n",
    "import sys\n",
    "import os\n",
    "import tensorflow as tf\n",
    "from transformers import DistilBertTokenizer\n",
    "from transformers import TFDistilBertForSequenceClassification\n",
    "from transformers import TextClassificationPipeline\n",
    "from transformers.configuration_distilbert import DistilBertConfig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')\n",
    "\n",
    "DATA_COLUMN = 'review_body'\n",
    "LABEL_COLUMN = 'star_rating'\n",
    "LABEL_VALUES = [0, 1, 2, 3, 4]\n",
    "    \n",
    "label_map = {}\n",
    "for (i, label) in enumerate(LABEL_VALUES):\n",
    "    label_map[label] = i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "class InputFeatures(object):\n",
    "  \"\"\"BERT feature vectors.\"\"\"\n",
    "\n",
    "  def __init__(self,\n",
    "               input_ids,\n",
    "               input_mask,\n",
    "               segment_ids,\n",
    "               label_id):\n",
    "    self.input_ids = input_ids\n",
    "    self.input_mask = input_mask\n",
    "    self.segment_ids = segment_ids\n",
    "    self.label_id = label_id\n",
    "    \n",
    "    \n",
    "class Input(object):\n",
    "  \"\"\"A single training/test input for sequence classification.\"\"\"\n",
    "\n",
    "  def __init__(self, text, label=None):\n",
    "    \"\"\"Constructs an Input.\n",
    "    Args:\n",
    "      text: string. The untokenized text of the first sequence. For single\n",
    "        sequence tasks, only this sequence must be specified.\n",
    "      label: (Optional) string. The label of the example. This should be\n",
    "        specified for train and dev examples, but not for test examples.\n",
    "    \"\"\"\n",
    "    self.text = text\n",
    "    self.label = label\n",
    "    \n",
    "    \n",
    "def convert_input(text_input, max_seq_length):\n",
    "    # First, we need to preprocess our data so that it matches the data BERT was trained on:\n",
    "    #\n",
    "    # 1. Lowercase our text (if we're using a BERT lowercase model)\n",
    "    # 2. Tokenize it (i.e. \"sally says hi\" -> [\"sally\", \"says\", \"hi\"])\n",
    "    # 3. Break words into WordPieces (i.e. \"calling\" -> [\"call\", \"##ing\"])\n",
    "    # \n",
    "    # Fortunately, the Transformers tokenizer does this for us!\n",
    "    #\n",
    "    tokens = tokenizer.tokenize(text_input.text)    \n",
    "\n",
    "    # Next, we need to do the following:\n",
    "    #\n",
    "    # 4. Map our words to indexes using a vocab file that BERT provides\n",
    "    # 5. Add special \"CLS\" and \"SEP\" tokens (see the [readme](https://github.com/google-research/bert))\n",
    "    # 6. Append \"index\" and \"segment\" tokens to each input (see the [BERT paper](https://arxiv.org/pdf/1810.04805.pdf))\n",
    "    #\n",
    "    # Again, the Transformers tokenizer does this for us!\n",
    "    #\n",
    "    encode_plus_tokens = tokenizer.encode_plus(text_input.text,\n",
    "                                               pad_to_max_length=True,\n",
    "                                               max_length=max_seq_length,\n",
    "#                                               truncation=True\n",
    "                                              )\n",
    "\n",
    "    # The id from the pre-trained BERT vocabulary that represents the token.  (Padding of 0 will be used if the # of tokens is less than `max_seq_length`)\n",
    "    input_ids = encode_plus_tokens['input_ids']\n",
    "    \n",
    "    # Specifies which tokens BERT should pay attention to (0 or 1).  Padded `input_ids` will have 0 in each of these vector elements.    \n",
    "    input_mask = encode_plus_tokens['attention_mask']\n",
    "\n",
    "    # Segment ids are always 0 for single-sequence tasks such as text classification.  1 is used for two-sequence tasks such as question/answer and next sentence prediction.\n",
    "    segment_ids = [0] * max_seq_length\n",
    "\n",
    "    # Label for each training row (`star_rating` 1 through 5)\n",
    "    label_id = label_map[text_input.label]\n",
    "\n",
    "    features = InputFeatures(\n",
    "        input_ids=input_ids,\n",
    "        input_mask=input_mask,\n",
    "        segment_ids=segment_ids,\n",
    "        label_id=label_id)\n",
    "\n",
    "#    print('**tokens**\\n{}\\n'.format(tokens))    \n",
    "#    print('**input_ids**\\n{}\\n'.format(features.input_ids))\n",
    "#    print('**input_mask**\\n{}\\n'.format(features.input_mask))\n",
    "#    print('**segment_ids**\\n{}\\n'.format(features.segment_ids))\n",
    "#    print('**label_id**\\n{}\\n'.format(features.label_id))\n",
    "\n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_features_to_tfrecord(inputs,\n",
    "                                 output_file,\n",
    "                                 max_seq_length):\n",
    "    \"\"\"Convert a set of `Input`s to a TFRecord file.\"\"\"\n",
    "\n",
    "    tfrecord_writer = tf.io.TFRecordWriter(output_file)\n",
    "\n",
    "    for (input_idx, text_input) in enumerate(inputs):\n",
    "        if input_idx % 1000 == 0:\n",
    "            print(\"Writing example %d of %d\" % (input_idx, len(inputs)))\n",
    "\n",
    "            bert_features = convert_input(text_input, max_seq_length)\n",
    "        \n",
    "            tfrecord_features = collections.OrderedDict()\n",
    "            \n",
    "            tfrecord_features['input_ids'] = tf.train.Feature(int64_list=tf.train.Int64List(value=bert_features.input_ids))\n",
    "            tfrecord_features['input_mask'] = tf.train.Feature(int64_list=tf.train.Int64List(value=bert_features.input_mask))\n",
    "            tfrecord_features['segment_ids'] = tf.train.Feature(int64_list=tf.train.Int64List(value=bert_features.segment_ids))\n",
    "            tfrecord_features['label_ids'] = tf.train.Feature(int64_list=tf.train.Int64List(value=[bert_features.label_id]))\n",
    "\n",
    "            tfrecord = tf.train.Example(features=tf.train.Features(feature=tfrecord_features))\n",
    "            \n",
    "            tfrecord_writer.write(tfrecord.SerializeToString())\n",
    "\n",
    "    tfrecord_writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no stored variable or alias max_seq_length\n"
     ]
    }
   ],
   "source": [
    "%store -r max_seq_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_seq_length=64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    max_seq_length\n",
    "except NameError:\n",
    "    print('++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++')\n",
    "    print('[ERROR] Please run the notebooks in the PREPARE section before you continue.')\n",
    "    print('++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "64\n"
     ]
    }
   ],
   "source": [
    "print(max_seq_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = './data-pipeline/data.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "filename_without_extension = Path(Path(file).stem).stem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "df = pd.read_csv(file, \n",
    "                 delimiter=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of dataframe (31005, 2)\n"
     ]
    }
   ],
   "source": [
    "print('Shape of dataframe {}'.format(df.shape))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TODO:  Make sure our pipeline up to now (ie. SM Data Wrangler) outputs the correct target label (ie. 0-4, 1-5, -1/0/1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train=df\n",
    "df_validation=df\n",
    "df_test=df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       star_rating                                        review_body\n",
      "0                4          Excellent. Looking forward to Windows 10.\n",
      "1                3  I have used various versions over Norton over ...\n",
      "2                1  I do not like the new budget outline it's very...\n",
      "3                1  Quicken 2015 release 8 will not recognize or r...\n",
      "4                4  These dopes think I'm going to write something...\n",
      "...            ...                                                ...\n",
      "31000            4                      Quicken 2015 works very well.\n",
      "31001            3  Good cleaner software for my laptop, works bet...\n",
      "31002            2  I am  not real happy with what I have. I keep ...\n",
      "31003            3  I have been satisfied with the performance of ...\n",
      "31004            1  I've used Quicken for years and I'm so terribl...\n",
      "\n",
      "[31005 rows x 2 columns]\n"
     ]
    }
   ],
   "source": [
    "print(df_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       star_rating                                        review_body\n",
      "0                4          Excellent. Looking forward to Windows 10.\n",
      "1                3  I have used various versions over Norton over ...\n",
      "2                1  I do not like the new budget outline it's very...\n",
      "3                1  Quicken 2015 release 8 will not recognize or r...\n",
      "4                4  These dopes think I'm going to write something...\n",
      "...            ...                                                ...\n",
      "31000            4                      Quicken 2015 works very well.\n",
      "31001            3  Good cleaner software for my laptop, works bet...\n",
      "31002            2  I am  not real happy with what I have. I keep ...\n",
      "31003            3  I have been satisfied with the performance of ...\n",
      "31004            1  I've used Quicken for years and I'm so terribl...\n",
      "\n",
      "[31005 rows x 2 columns]\n"
     ]
    }
   ],
   "source": [
    "print(df_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       star_rating                                        review_body\n",
      "0                4          Excellent. Looking forward to Windows 10.\n",
      "1                3  I have used various versions over Norton over ...\n",
      "2                1  I do not like the new budget outline it's very...\n",
      "3                1  Quicken 2015 release 8 will not recognize or r...\n",
      "4                4  These dopes think I'm going to write something...\n",
      "...            ...                                                ...\n",
      "31000            4                      Quicken 2015 works very well.\n",
      "31001            3  Good cleaner software for my laptop, works bet...\n",
      "31002            2  I am  not real happy with what I have. I keep ...\n",
      "31003            3  I have been satisfied with the performance of ...\n",
      "31004            1  I've used Quicken for years and I'm so terribl...\n",
      "\n",
      "[31005 rows x 2 columns]\n"
     ]
    }
   ],
   "source": [
    "print(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_inputs = df_train.apply(lambda x: Input(text = x[DATA_COLUMN],\n",
    "                                              label = x[LABEL_COLUMN]), axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_inputs = df_validation.apply(lambda x: Input(text = x[DATA_COLUMN], \n",
    "                                                        label = x[LABEL_COLUMN]), axis = 1)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_inputs = df_test.apply(lambda x: Input(text = x[DATA_COLUMN], \n",
    "                                            label = x[LABEL_COLUMN]), axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfrecord_output_path='./output'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing example 0 of 31005\n",
      "Writing example 1000 of 31005\n",
      "Writing example 2000 of 31005\n",
      "Writing example 3000 of 31005\n",
      "Writing example 4000 of 31005\n",
      "Writing example 5000 of 31005\n",
      "Writing example 6000 of 31005\n",
      "Writing example 7000 of 31005\n",
      "Writing example 8000 of 31005\n",
      "Writing example 9000 of 31005\n",
      "Writing example 10000 of 31005\n",
      "Writing example 11000 of 31005\n",
      "Writing example 12000 of 31005\n",
      "Writing example 13000 of 31005\n",
      "Writing example 14000 of 31005\n",
      "Writing example 15000 of 31005\n",
      "Writing example 16000 of 31005\n",
      "Writing example 17000 of 31005\n",
      "Writing example 18000 of 31005\n",
      "Writing example 19000 of 31005\n",
      "Writing example 20000 of 31005\n",
      "Writing example 21000 of 31005\n",
      "Writing example 22000 of 31005\n",
      "Writing example 23000 of 31005\n",
      "Writing example 24000 of 31005\n",
      "Writing example 25000 of 31005\n",
      "Writing example 26000 of 31005\n",
      "Writing example 27000 of 31005\n",
      "Writing example 28000 of 31005\n",
      "Writing example 29000 of 31005\n",
      "Writing example 30000 of 31005\n",
      "Writing example 31000 of 31005\n",
      "Writing example 0 of 31005\n",
      "Writing example 1000 of 31005\n",
      "Writing example 2000 of 31005\n",
      "Writing example 3000 of 31005\n",
      "Writing example 4000 of 31005\n",
      "Writing example 5000 of 31005\n",
      "Writing example 6000 of 31005\n",
      "Writing example 7000 of 31005\n",
      "Writing example 8000 of 31005\n",
      "Writing example 9000 of 31005\n",
      "Writing example 10000 of 31005\n",
      "Writing example 11000 of 31005\n",
      "Writing example 12000 of 31005\n",
      "Writing example 13000 of 31005\n",
      "Writing example 14000 of 31005\n",
      "Writing example 15000 of 31005\n",
      "Writing example 16000 of 31005\n",
      "Writing example 17000 of 31005\n",
      "Writing example 18000 of 31005\n",
      "Writing example 19000 of 31005\n",
      "Writing example 20000 of 31005\n",
      "Writing example 21000 of 31005\n",
      "Writing example 22000 of 31005\n",
      "Writing example 23000 of 31005\n",
      "Writing example 24000 of 31005\n",
      "Writing example 25000 of 31005\n",
      "Writing example 26000 of 31005\n",
      "Writing example 27000 of 31005\n",
      "Writing example 28000 of 31005\n",
      "Writing example 29000 of 31005\n",
      "Writing example 30000 of 31005\n",
      "Writing example 31000 of 31005\n",
      "Writing example 0 of 31005\n",
      "Writing example 1000 of 31005\n",
      "Writing example 2000 of 31005\n",
      "Writing example 3000 of 31005\n",
      "Writing example 4000 of 31005\n",
      "Writing example 5000 of 31005\n",
      "Writing example 6000 of 31005\n",
      "Writing example 7000 of 31005\n",
      "Writing example 8000 of 31005\n",
      "Writing example 9000 of 31005\n",
      "Writing example 10000 of 31005\n",
      "Writing example 11000 of 31005\n",
      "Writing example 12000 of 31005\n",
      "Writing example 13000 of 31005\n",
      "Writing example 14000 of 31005\n",
      "Writing example 15000 of 31005\n",
      "Writing example 16000 of 31005\n",
      "Writing example 17000 of 31005\n",
      "Writing example 18000 of 31005\n",
      "Writing example 19000 of 31005\n",
      "Writing example 20000 of 31005\n",
      "Writing example 21000 of 31005\n",
      "Writing example 22000 of 31005\n",
      "Writing example 23000 of 31005\n",
      "Writing example 24000 of 31005\n",
      "Writing example 25000 of 31005\n",
      "Writing example 26000 of 31005\n",
      "Writing example 27000 of 31005\n",
      "Writing example 28000 of 31005\n",
      "Writing example 29000 of 31005\n",
      "Writing example 30000 of 31005\n",
      "Writing example 31000 of 31005\n"
     ]
    }
   ],
   "source": [
    "train_data = '{}/bert/train'.format(tfrecord_output_path)\n",
    "validation_data = '{}/bert/validation'.format(tfrecord_output_path)\n",
    "test_data = '{}/bert/test'.format(tfrecord_output_path)\n",
    "\n",
    "# Convert our train and validation features to InputFeatures (.tfrecord protobuf) that works with BERT and TensorFlow.\n",
    "df_train_embeddings = convert_features_to_tfrecord(train_inputs, \n",
    "                                                   '{}/part-{}-{}.tfrecord'.format(train_data, 'algo-1', filename_without_extension), \n",
    "                                                   max_seq_length)\n",
    "\n",
    "df_validation_embeddings = convert_features_to_tfrecord(validation_inputs, \n",
    "                                                        '{}/part-{}-{}.tfrecord'.format(validation_data, 'algo-1', filename_without_extension), \n",
    "                                                        max_seq_length)\n",
    "\n",
    "df_test_embeddings = convert_features_to_tfrecord(test_inputs, \n",
    "                                                  '{}/part-{}-{}.tfrecord'.format(test_data, 'algo-1', filename_without_extension), \n",
    "                                                  max_seq_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "def select_data_and_label_from_record(record):\n",
    "    x = {\n",
    "        'input_ids': record['input_ids'],\n",
    "        'input_mask': record['input_mask'],\n",
    "        'segment_ids': record['segment_ids']\n",
    "    }\n",
    "    y = record['label_ids']\n",
    "\n",
    "    return (x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "def file_based_input_dataset_builder(channel,\n",
    "                                     input_filenames,\n",
    "                                     pipe_mode,\n",
    "                                     is_training,\n",
    "                                      drop_remainder):\n",
    "\n",
    "    # For training, we want a lot of parallel reading and shuffling.\n",
    "    # For eval, we want no shuffling and parallel reading doesn't matter.\n",
    "\n",
    "    if pipe_mode:\n",
    "        print('***** Using pipe_mode with channel {}'.format(channel))\n",
    "        from sagemaker_tensorflow import PipeModeDataset\n",
    "        dataset = PipeModeDataset(channel=channel,\n",
    "                                  record_format='TFRecord')\n",
    "    else:\n",
    "        print('***** Using input_filenames {}'.format(input_filenames))\n",
    "        dataset = tf.data.TFRecordDataset(input_filenames)\n",
    "\n",
    "    dataset = dataset.repeat(100)\n",
    "    dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)\n",
    "\n",
    "    name_to_features = {\n",
    "      \"input_ids\": tf.io.FixedLenFeature([max_seq_length], tf.int64),\n",
    "      \"input_mask\": tf.io.FixedLenFeature([max_seq_length], tf.int64),\n",
    "      \"segment_ids\": tf.io.FixedLenFeature([max_seq_length], tf.int64),\n",
    "      \"label_ids\": tf.io.FixedLenFeature([], tf.int64),\n",
    "    }\n",
    "\n",
    "    def _decode_record(record, name_to_features):\n",
    "        \"\"\"Decodes a record to a TensorFlow example.\"\"\"\n",
    "        return tf.io.parse_single_example(record, name_to_features)\n",
    "        \n",
    "    dataset = dataset.apply(\n",
    "        tf.data.experimental.map_and_batch(\n",
    "          lambda record: _decode_record(record, name_to_features),\n",
    "          batch_size=8,\n",
    "          drop_remainder=drop_remainder,\n",
    "          num_parallel_calls=tf.data.experimental.AUTOTUNE))\n",
    "\n",
    "    dataset.cache()\n",
    "\n",
    "    if is_training:\n",
    "        dataset = dataset.shuffle(seed=42,\n",
    "                                  buffer_size=10,\n",
    "                                  reshuffle_each_iteration=True)\n",
    "\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_data_filenames ['./output/bert/train/part-algo-1-data.tfrecord']\n",
      "***** Using input_filenames ['./output/bert/train/part-algo-1-data.tfrecord']\n",
      "WARNING:tensorflow:From <ipython-input-46-e97a3ab17503>:38: map_and_batch (from tensorflow.python.data.experimental.ops.batching) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.data.Dataset.map(map_func, num_parallel_calls)` followed by `tf.data.Dataset.batch(batch_size, drop_remainder)`. Static tf.data optimizations will take care of using the fused implementation.\n"
     ]
    }
   ],
   "source": [
    "#train_data = './data-tfrecord/bert-train'\n",
    "\n",
    "train_data = './output/bert/train'\n",
    "\n",
    "train_data_filenames = glob('{}/*.tfrecord'.format(train_data))\n",
    "print('train_data_filenames {}'.format(train_data_filenames))\n",
    "\n",
    "train_dataset = file_based_input_dataset_builder(\n",
    "    channel='train',\n",
    "    input_filenames=train_data_filenames,\n",
    "    pipe_mode=False,\n",
    "    is_training=True,\n",
    "    drop_remainder=False).map(select_data_and_label_from_record)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "validation_data_filenames ['./output/bert/validation/part-algo-1-data.tfrecord']\n",
      "***** Using input_filenames ['./output/bert/validation/part-algo-1-data.tfrecord']\n"
     ]
    }
   ],
   "source": [
    "# validation_data = './data-tfrecord/bert-validation'\n",
    "\n",
    "validation_data = './output/bert/validation'\n",
    "validation_data_filenames = glob('{}/*.tfrecord'.format(validation_data))\n",
    "print('validation_data_filenames {}'.format(validation_data_filenames))\n",
    "\n",
    "validation_dataset = file_based_input_dataset_builder(\n",
    "    channel='validation',\n",
    "    input_filenames=validation_data_filenames,\n",
    "    pipe_mode=False,\n",
    "    is_training=False,\n",
    "    drop_remainder=False).map(select_data_and_label_from_record)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['./output/bert/test/part-algo-1-data.tfrecord']\n",
      "***** Using input_filenames ['./output/bert/test/part-algo-1-data.tfrecord']\n"
     ]
    }
   ],
   "source": [
    "#test_data = './data-tfrecord/bert-test'\n",
    "\n",
    "test_data = './output/bert/test'\n",
    "test_data_filenames = glob('{}/*.tfrecord'.format(test_data))\n",
    "print(test_data_filenames)\n",
    "\n",
    "test_dataset = file_based_input_dataset_builder(\n",
    "    channel='test',\n",
    "    input_filenames=test_data_filenames,\n",
    "    pipe_mode=False,\n",
    "    is_training=False,\n",
    "    drop_remainder=False).map(select_data_and_label_from_record)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Specify Manual Hyper-Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs=1\n",
    "steps_per_epoch=5\n",
    "validation_steps=5\n",
    "test_steps=5\n",
    "freeze_bert_layer=True\n",
    "learning_rate=3e-5\n",
    "epsilon=1e-08"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Pretrained BERT Model \n",
    "https://huggingface.co/transformers/pretrained_models.html "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c1ae879b6c00405b948b83f859c9fd8e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=442.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "61385a8afa8d4c558dbd35dfbd40aef6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=363423424.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "CLASSES=[1, 2, 3, 4, 5]\n",
    "\n",
    "config = DistilBertConfig.from_pretrained('distilbert-base-uncased',\n",
    "                                          num_labels=len(CLASSES))\n",
    "\n",
    "model = TFDistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased', \n",
    "                                                              config=config)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup the Custom Classifier Model Here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"tf_distil_bert_for_sequence_classification\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "distilbert (TFDistilBertMain multiple                  66362880  \n",
      "_________________________________________________________________\n",
      "pre_classifier (Dense)       multiple                  590592    \n",
      "_________________________________________________________________\n",
      "classifier (Dense)           multiple                  3845      \n",
      "_________________________________________________________________\n",
      "dropout_19 (Dropout)         multiple                  0         \n",
      "=================================================================\n",
      "Total params: 66,957,317\n",
      "Trainable params: 594,437\n",
      "Non-trainable params: 66,362,880\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
    "metric=tf.keras.metrics.SparseCategoricalAccuracy('accuracy')\n",
    "\n",
    "optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate, epsilon=epsilon)\n",
    "model.compile(optimizer=optimizer, loss=loss, metrics=[metric])\n",
    "model.layers[0].trainable=not freeze_bert_layer\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "callbacks = []\n",
    "\n",
    "log_dir = './tmp/tensorboard/'\n",
    "tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir)\n",
    "callbacks.append(tensorboard_callback)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train for 5 steps, validate for 5 steps\n",
      "5/5 [==============================] - 7s 1s/step - loss: 1.6103 - accuracy: 0.1500 - val_loss: 1.5975 - val_accuracy: 0.3250\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(train_dataset,\n",
    "                    shuffle=True,\n",
    "                    epochs=epochs,\n",
    "                    steps_per_epoch=steps_per_epoch,\n",
    "                    validation_data=validation_dataset,\n",
    "                    validation_steps=validation_steps,\n",
    "                    callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trained model <transformers.modeling_tf_distilbert.TFDistilBertForSequenceClassification object at 0x7fb110388da0>\n"
     ]
    }
   ],
   "source": [
    "print('Trained model {}'.format(model))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluate on Holdout Test Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5/5 [==============================] - 1s 124ms/step - loss: 1.5975 - accuracy: 0.3250\n",
      "[1.5975335359573364, 0.325]\n"
     ]
    }
   ],
   "source": [
    "test_history = model.evaluate(test_dataset,\n",
    "                              steps=test_steps,                            \n",
    "                              callbacks=callbacks)\n",
    "print(test_history)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_dir = './tmp/fine-tuned'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir -p $model_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save_pretrained(model_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 261692\n",
      "drwxrwxr-x 2 ec2-user ec2-user      4096 Dec 14 01:54 .\n",
      "drwxr-xr-x 4 ec2-user ec2-user      4096 Dec 14 01:54 ..\n",
      "-rw-rw-r-- 1 ec2-user ec2-user      1358 Dec 14 01:54 config.json\n",
      "-rw-rw-r-- 1 ec2-user ec2-user 267959068 Dec 14 01:54 tf_model.h5\n"
     ]
    }
   ],
   "source": [
    "!ls -al $model_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"_num_labels\": 5,\n",
      "  \"activation\": \"gelu\",\n",
      "  \"architectures\": [\n",
      "    \"DistilBertForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_dropout\": 0.1,\n",
      "  \"bad_words_ids\": null,\n",
      "  \"bos_token_id\": null,\n",
      "  \"decoder_start_token_id\": null,\n",
      "  \"dim\": 768,\n",
      "  \"do_sample\": false,\n",
      "  \"dropout\": 0.1,\n",
      "  \"early_stopping\": false,\n",
      "  \"eos_token_id\": null,\n",
      "  \"finetuning_task\": null,\n",
      "  \"hidden_dim\": 3072,\n",
      "  \"id2label\": {\n",
      "    \"0\": \"LABEL_0\",\n",
      "    \"1\": \"LABEL_1\",\n",
      "    \"2\": \"LABEL_2\",\n",
      "    \"3\": \"LABEL_3\",\n",
      "    \"4\": \"LABEL_4\"\n",
      "  },\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"is_decoder\": false,\n",
      "  \"is_encoder_decoder\": false,\n",
      "  \"label2id\": {\n",
      "    \"LABEL_0\": 0,\n",
      "    \"LABEL_1\": 1,\n",
      "    \"LABEL_2\": 2,\n",
      "    \"LABEL_3\": 3,\n",
      "    \"LABEL_4\": 4\n",
      "  },\n",
      "  \"length_penalty\": 1.0,\n",
      "  \"max_length\": 20,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"min_length\": 0,\n",
      "  \"model_type\": \"distilbert\",\n",
      "  \"n_heads\": 12,\n",
      "  \"n_layers\": 6,\n",
      "  \"no_repeat_ngram_size\": 0,\n",
      "  \"num_beams\": 1,\n",
      "  \"num_return_sequences\": 1,\n",
      "  \"output_attentions\": false,\n",
      "  \"output_hidden_states\": false,\n",
      "  \"output_past\": true,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"prefix\": null,\n",
      "  \"pruned_heads\": {},\n",
      "  \"qa_dropout\": 0.1,\n",
      "  \"repetition_penalty\": 1.0,\n",
      "  \"seq_classif_dropout\": 0.2,\n",
      "  \"sinusoidal_pos_embds\": false,\n",
      "  \"task_specific_params\": null,\n",
      "  \"temperature\": 1.0,\n",
      "  \"tie_weights_\": true,\n",
      "  \"top_k\": 50,\n",
      "  \"top_p\": 1.0,\n",
      "  \"torchscript\": false,\n",
      "  \"use_bfloat16\": false,\n",
      "  \"vocab_size\": 30522\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "cat $model_dir/config.json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predict with Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from transformers import TFDistilBertForSequenceClassification\n",
    "\n",
    "loaded_model = TFDistilBertForSequenceClassification.from_pretrained(model_dir,\n",
    "                                                                     id2label={\n",
    "                                                                       0: 1,\n",
    "                                                                       1: 2,\n",
    "                                                                       2: 3,\n",
    "                                                                       3: 4,\n",
    "                                                                       4: 5\n",
    "                                                                     },\n",
    "                                                                     label2id={\n",
    "                                                                       1: 0,\n",
    "                                                                       2: 1,\n",
    "                                                                       3: 2,\n",
    "                                                                       4: 3,\n",
    "                                                                       5: 4\n",
    "                                                                     })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from transformers import DistilBertTokenizer\n",
    "\n",
    "tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I loved it!  I will recommend this to everyone. [{'label': 2, 'score': 0.20833898}]\n",
      "Really bad.  I hope they don't make this anymore. [{'label': 2, 'score': 0.20828648}]\n"
     ]
    }
   ],
   "source": [
    "from transformers import TextClassificationPipeline\n",
    "inference_pipeline = TextClassificationPipeline(model=loaded_model, \n",
    "                                                tokenizer=tokenizer,\n",
    "                                                framework='tf',\n",
    "                                                device=-1) # -1 is CPU, >= 0 is GPU\n",
    "\n",
    "print(\"\"\"I loved it!  I will recommend this to everyone.\"\"\", inference_pipeline(\"\"\"I loved it!  I will recommend this to everyone.\"\"\"))\n",
    "print(\"\"\"Really bad.  I hope they don't make this anymore.\"\"\", inference_pipeline(\"\"\"Really bad.  I hope they don't make this anymore.\"\"\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100, 2)"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import csv\n",
    "\n",
    "df_sample_reviews = pd.read_csv('./data/amazon_reviews_us_Digital_Software_v1_00.tsv.gz', \n",
    "                                delimiter='\\t', \n",
    "                                quoting=csv.QUOTE_NONE,\n",
    "                                compression='gzip')[['review_body', 'star_rating']].sample(n=100)\n",
    "df_sample_reviews.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review_body</th>\n",
       "      <th>star_rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>72609</th>\n",
       "      <td>Quicken (Intuit) produces good product and Qui...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75760</th>\n",
       "      <td>If they would have told me that the data forma...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34173</th>\n",
       "      <td>If you want a basic photo editor and don't wan...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28320</th>\n",
       "      <td>Smooth buy</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52696</th>\n",
       "      <td>I bought PC Matic when Microsoft stopped suppo...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             review_body  star_rating\n",
       "72609  Quicken (Intuit) produces good product and Qui...            4\n",
       "75760  If they would have told me that the data forma...            1\n",
       "34173  If you want a basic photo editor and don't wan...            2\n",
       "28320                                         Smooth buy            5\n",
       "52696  I bought PC Matic when Microsoft stopped suppo...            5"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_sample_reviews.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "72609    2\n",
       "75760    2\n",
       "34173    2\n",
       "28320    2\n",
       "52696    2\n",
       "        ..\n",
       "82030    2\n",
       "27130    2\n",
       "56544    2\n",
       "73310    3\n",
       "53829    1\n",
       "Name: review_body, Length: 100, dtype: int64"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "def predict(review_body):\n",
    "    prediction_map = inference_pipeline(review_body)\n",
    "    return prediction_map[0]['label']\n",
    "    \n",
    "y_pred = df_sample_reviews['review_body'].map(predict)\n",
    "\n",
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "72609    4\n",
       "75760    1\n",
       "34173    2\n",
       "28320    5\n",
       "52696    5\n",
       "        ..\n",
       "82030    5\n",
       "27130    4\n",
       "56544    5\n",
       "73310    4\n",
       "53829    4\n",
       "Name: star_rating, Length: 100, dtype: int64"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_true = df_sample_reviews['star_rating']\n",
    "\n",
    "y_true"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classification Report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           1       0.20      0.04      0.07        24\n",
      "           2       0.04      0.60      0.07         5\n",
      "           3       0.00      0.00      0.00         9\n",
      "           4       0.00      0.00      0.00        13\n",
      "           5       0.00      0.00      0.00        49\n",
      "\n",
      "    accuracy                           0.04       100\n",
      "   macro avg       0.05      0.13      0.03       100\n",
      "weighted avg       0.05      0.04      0.02       100\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages/sklearn/metrics/_classification.py:1221: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "\n",
    "print(classification_report(y_true=y_true, y_pred=y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy:  0.04\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "print('Test Accuracy: ', accuracy_score(y_pred=y_pred, y_true=y_true))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sn\n",
    "import pandas as pd\n",
    "\n",
    "def plot_conf_mat(cm, classes, title, cmap = plt.cm.Greens):\n",
    "    print(cm)\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes, rotation=45)\n",
    "    plt.yticks(tick_marks, classes)\n",
    "\n",
    "    fmt = 'd'\n",
    "    thresh = cm.max() / 2.\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        plt.text(j, i, format(cm[i, j], fmt),\n",
    "        horizontalalignment=\"center\",\n",
    "        color=\"black\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "        plt.tight_layout()\n",
    "        plt.ylabel('True label')\n",
    "        plt.xlabel('Predicted label')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1 21  2  0  0]\n",
      " [ 0  3  2  0  0]\n",
      " [ 2  7  0  0  0]\n",
      " [ 1 10  2  0  0]\n",
      " [ 1 42  6  0  0]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAyAAAAK+CAYAAABaa3rvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3XecXHW5+PHPkwTBBBJCgIQOCSmotChIbwFELoKoeBUV8doREQUbRSMglotKExBUkI5XfjSvEukB6QlFRGkhFBUCBBII5Ury/P44Z2Gz2WxmdyZnMpPP29e8Zuec7znn2TlhnWeeb4nMRJIkSZKq0K/ZAUiSJElaepiASJIkSaqMCYgkSZKkypiASJIkSaqMCYgkSZKkypiASJIkSaqMCYgkSZKkypiASJIkSaqMCYgkSZKkypiASJIkSaqMCYgkSZKkypiASJIkSaqMCYgkSZKkypiASGoLEbFZRFwREc9GxLyIyIiY2IQ41i2vnVVfW/PzXkjSkskERNICImJgRHyx/ED/eES8HBFzIuLRiPhdRHw8It7a7Dg7RMRo4HpgD2Ao8CzwNPBSE8NqGRExveODekTcW0P7n3VqnxGxbgNj2SEiJkbE+xt1TknSkmVAswOQtGSJiPcBpwMjOm2eA8wD1i0fHwR+FBGfyMxrq46xG58DBgI3Antm5gtNjOXfwANNvH69NoyITTLz7u52RsQAYN/FeP0dgO8CvwEurfNcrX4vJKktWQGR9IaI2J/iQ98Iig9unwBWzszlM3MwsCLwIYpqw+rAds2JdAFvL59/2+Tkg8z8R2aOy8xxzYyjjx4vn/froc1uwKrAY4s/nPq0+L2QpLZlAiIJgIjYGDiN4u/CH4BNM/PczHyuo01mzsrMizNzR+AjwIvNiXYBHd3B7HJVn/OABD4aEf0X0qYjOTm3mpAkSe3GBERSh2OAZYF/APtm5is9Nc7Mi4Cfdt0eEctGxNci4raImBURr0TEAxHx04gY0c2piIj9y7EE15ev3xcR10XECxHxUkTcGhEf7ea46eUA4x3KTWd2GpcwvVO7Hscq9DRYOSL6lfFdFxHPRcS/I+KZiPhrRPw6Inar9Vyd2mwaEedGxBMR8Vo5cH5SRHywh2M6xmnsEBErle/no+Xx/4iIMyJitYUdX6PHgRsoKmC7dhPDisD7gJeB3/V0oojYLiJOKP8d/DMi/i8iZkTElRHxoW7ar1u+Z98tN32yyziTN+5f1/c4IrYoxyb9KyLmRsTx3bXrdK33lBMVzIuIBX7Pss23y2NnNXKMiyTJMSCSgIhYA/iP8uWJmTmrluMys+sHu1WAScCm5abXgP8DxpSP/SNi98y8tYdYjgSOohhz8iIwCHg3cH5EDM/M4zs1fwZYDlgJWAaYDbzSaV8jnMP8Yx5mAYOBlYG3lY8raz1ZRHwOOJU3vwB6gaJr267ArhFxLrB/Zs5dyCnWBM4C1qFIBJKiO9xngJ0jYnxmPl9rPN04hyKh2w/4Y5d9H6Z4v8+nh2pTRCxPkch0eJHivqwCvAd4T0Scnpmf79RmLsXEActT3PNXKd5rurTpeq3/pKjGDCjbL+x9e0NmToqInwMHUiStG2bmzE7n3BT4XvnyK5k5fVHnlCTVzgqIJCg+cEb58+V1nOdsiuTjeYoPq4PKsSObAX+hmKHq0ohYeSHHb0LxDfiRwLDMXJHi2/iOb9t/EBErdTTOzM0ycwRwc7npK5k5onxsVsfvARTf4lMkH3OBrwKDy5iWo/jQvz9wUy/OtxVvJh+/A9bKzKEUCcgRFMnEx4Fv93Cakyje360ycxDFB/a9KBKZdRdxbC3+hyJZ2CsiBnfZ19H96uxFnGMexe+3N8V9HJyZQyju/4EUycvnImKfjgMy84nyXh5Xbrqo073seDzRzbV+CVwGrFfem4HA8d206+obwN8p7uNpHRsjYjmKhGYZ4P9l5lk1nEuS1AsmIJIANiifX6OPswZFxLYUA5QBPpqZ/9PxLX5m3gnsQvHBeThw0EJOMwT4bmYe0zGYPDOfpvjg21Ht2KMv8fXRFuXzVZl5fGa+WMaUmfmvzPxNZh7ai/MdTfF398/ARzLzyfJ8L2Xm94Eflu2+2c2H/w6vATtn5i3lsa9n5uUUXeigmCSgz8rf8VKKcTVvnCsiRgJbA/8Crl7EOV7OzH0y89LOlYXMfCEzfw4cUG46oPsz9Mo9wIc7qhTl+zF9UQeVXQw/RjFT1j4R8Yly1w8pqlpPUcyuJklqMBMQSQDDyufnu3ar6oWOD6t3ZuakrjvLRKLjm+YPL+Qcr9LNt9flh8WOc76jj/H1xezyedWIqOvvZVm52bF8+YOFdLH6EcV7sDyw+0JOdXrniQE66Ziydr2IGFRPrLxZ4eg8G1bHz+f10D2sVleUz1v0MNi9Vj/JzHl9OTAzp/LmmJOTI+K/eDM5/q+FvM+SpDqZgEhqlPHl83U9tOlYM2TMQj4k35+ZcxZy7D/K56F9Ca6PrqEYwzIeuD6KBRhX7+O5NqXo5pbMPz7iDeXYmynly/HdtQHuWMj2f3T6ecW+BNjJVRSVju0iYp1yW0eFYFHdr4BivZCI+HQ56Pxf5WD5jgHhHWNUlqP++3lLncf/iKIb3WDgVxT36NTM7Dr+RZLUICYgkgA6vukdGhHRY8uFW6V8/kcPbZ4sn4NiEHdXPU3r+2r5vEwv4+qzzHwI+CLFmIhtKQZo/6OcferUcrByrTren1mZ2dN0wR3v0SoL2d/te5SZr3Z6Wdd7VFY4zqe4Tx+PiG2AkcDdmfmXRR3faRD6LykGnY+gGEfzDMVA86c7Na+3WlPXZANl9eQznTZNB3rTrU6S1EsmIJIA/lY+LwuMrfNcy9V5/BIlM38NrAccTDHY+TmKwd5fAKZExGG9POWyDQ1w8emodHyC2gefdzgS2Ap4FvgkMDwzB2bmquVA8zU6te1rwgu8kSzV61Odfl4NGNWAc0qSFsIERBIU31Z3jP3Ys4/n6Pgmeu0e2qxZPifFh9OqdHxIXVhyNKSngzPz6cw8ITPfT1GZ2By4hOLD89ERsVENMXS8P28tpytemI73qFHTCPdJZt4L3EuRkO5P8R6eX+PhHbNbfTkzz87MGV32D29IkA1QTp7w9fLlfRQJ4rkR8ZbmRSVJ7c0ERBLlbEx/KF9+uYcZmObTpbvW1PJ5+x66ce1UPj/Yw1iPxeGF8nnNheyvecrecgasOyg+ZD9J8Xd0mxoOvYs3k7wdu2sQEUOAd5Yvp3bXpmIdFY9lgD+VEwnUouN9vmsh+3fu4diOAeV1VUZqUf47P5viHv6a4t/nDGAj3pxVTJLUYCYgkjocQTHF65oUi/712JUqIj4MfK3Tpo61Ot5OsS5F1/bDKbotAfy27mh7p2PcQndxLUvRvWoBPX0LXnb9+Xf5cpHdqsrpaDsG6H9zIbNqfZOiSvMSbyaEzXQO8JPy8f1eHNexgOCGXXeU40MO7+HYjpnH6h1IX4sTKbrTPQocnJnPAJ8t9x1SrgMjSWowExBJAGTm3cCXKL6l/w/grnLWpzcW/ouIIRHxgYi4DrgIWKHT8Tfy5orgv46ID3VMsRoR7wT+RDHj0dPACVX8Tp10JDyfjYhPlUkHEfF2ig/6C5vZ6tiI+F1EvL/L+zA8Ik6kGBuSFLNG1eJIim/4xwMXRsSa5fmWL8eSfKts98PMnL2Qc1QmM2dk5qHl48+9OLTj/fhpRLxREYuIzShmFhu20CPhr+XzNhExuvdR1yYiPkAxPmUe8IlOa7xcTjEbVj/gN7VWAyVJtTMBkfSGzPwV8AGKbijjKL4Bfy4iXoyI2RRdmS6mWDn9Md6cVrfDfsDdFInG/wAvlcfdSdGt5Xlg7yasr/BL4DaKSsWvy7hmUfT534T5ByF3NgD4IMV4j+ciYlb5+zwFfLlsc0Rm3ldLEJl5M8Xie/MounA9HhEzKd7X71N0OzqPNxckbFVHUIzxWQu4Hng5Il4Cbqeoiuzbw7HXA48AKwEPRMSMiJhePhbWha5XImIEcHr58sfdJFcHA9MoqiMnNuKakqQ3mYBImk9mXkox5eqXKKoDT1J8EB9AMUXp7yg+QI7NzMldjn0G2JJiGtM7KboovQV4iGKBwbd3rOBdpcz8N8VK7P9N8TvMA+YAZ1GMubhnIYf+jGJhusuABykShGWBJygqQNtl5rG9jOUXFGNOzqdYa2N5ii5LVwH7ZObHGzSzU9Nk5jSKgfrnUiSz/SmSrPOAzTLzTz0c+29gAuWUxxTJ7DrlY0CDQvw1RRXmbt5ciLBzDC9RJNPzgE+W1RJJUoNE3xc9liRJkqTesQIiSZIkqTImIJIkSZIqYwIiSZIkqTImIJIkSZIqYwIiSZIkqTImIJIkSZIqYwIiSZIkqTImIJIkSZIqYwIiSZIkqTImIJIkSZIqM6DZASxJIuJRYDAwvcmhSJIkNcu6wOzMXK/ZgSxJIuI8YFwTLv33zPxYE6672JiAzG/wW9/61pXGjhuzUrMDUWP8c85TzQ5BDbTaoBHNDkEN1C+i2SFI6sbf//YAr7zySrPDWBKNo1+MZ1CFH5/nvA7zsrrrVcQEZH7Tx44bs9KNt01udhxqkIm3HdPsENRAR27+rWaHoAZatv9yzQ5BUje22nxr7pp69/Rmx7FEGjQA3r1qdde7bQa8+O/qrlcRExBJkiSpVlZv6+YgdEmSJEmVMQGRJEmSVBm7YEmSJEm1CKr9+r5Ne3tZAZEkSZJUGSsgkiRJUq0chF43KyCSJEmSKmMFRJIkSaqVBZC6WQGRJEmSVBkrIJIkSVKtHANSNysgkiRJkipjAiJJkiSpMnbBkiRJkmrhQoQNYQVEkiRJUmWsgEiSJEk1iYoHobdnCcQKiCRJkqTKWAGRJEmSatWeRYlKWQGRJEmSVBkrIJIkSVKt+lkCqZcVEEmSJEmVMQGRJEmSliIR8fGIyPLxmYW02SMiro+IWRHxUkTcFhGfbMT1TUAkSZKkWkQTHo3+FSLWAk4GXuqhzYHAFcA7gHOBM4DVgbMi4rh6YzABkSRJkpYCERHAmcBzwGkLabMucBwwE3hXZn4pM78KbAQ8AhwSEVvWE4cJiCRJklSriOoejXcQsBPwKWDOQtr8F7AscHJmTu/YmJnPA8eWL79QTxDOgiVJkiQtucZFxJTudmTmO2s9SURsAPwQOCEzJ0fETgtp2rH9ym72/bFLmz4xAZEkSZJq1YKz8EbEAOAc4HHgsEU0H1s+P9h1R2b+KyLmAGtGxMDMfLkv8ZiASJIkSUuuv/em0rEQ3wE2BbbJzFcW0XZI+TxrIftnAYPKdiYgkiRJ0mLVYgsRRsS7KaoeP8nMW5odDzgIXZIkSWpLZdersym6Ux1Z42EdlY8hC9m/qArJIpmASJIkSe1peWAMsAHwaqfFBxP4btnmjHLb8eXrB8rnMV1PFhGrUXS/erKv4z/ALliSJElSbRbT4oA9Xq8+rwG/Wsi+8RTjQm6iSDo6umddC2wN7NZpW4f3dmrTZyYgkiRJUhsqB5x/prt9ETGRIgH5TWb+stOuM4FvAAdGxJkda4FExFDenEGr20UMa2UCIkmSJNVq8SwQuMTIzEcj4uvAicCdEXER8H/Ah4A1acBgdhMQSZIkSW/IzJMiYjpwKLAfxbjx+4EjMvM39Z7fBESSJEmqSVQ8De/iu1ZmTgQm9rD/CuCKxXFtZ8GSJEmSVBkrIJIkSVKt2nsISCWsgEiSJEmqjAmIJEmSpMrYBUuSJEmqRVDtNLxt2t3LCogkSZKkylgBkSRJkmrVplWJKpmALCUuvfhSbrrxz9x7z1+47977ePHFF/nPj36YX/7mjGaHpoV4ZdYrPDT5IR695RGenfYsLz3zEv2W6cfKI1fhHbu/g3fsviHRaS7yua/P5Z5L7mbGQzOY8dDTPDf9Oea9Po9dvvEeNnrfRk38TdSTmc/N5PeX/S+T/vgn7r/vb/zrn//iLW9Zhre9423su9++fPyT+9Kvn8XqVvPkk//g6IlH86dJVzHzuZmMWG0E79trDw4/8jCGDh3a7PDUS95PqbFMQJYSP/7Bcfzl3r+w/PLLs/oaq/PiAy82OyQtwoPXPcDVP7mKQcMGsdb4tRm86mDmPD+Hhyc/xJ9+NIlHb32U9x29J1H2Rf33K//muhOvBWDgSgMZtNIgXpzhfV7SXXrxZXzty4cyYrXhbLv9tqy51hrMmPEMV1z6ew76wle4etLV/OaCM9+4z1ryTXtkGjtuuxMzZjzDHnvuwdixY7jzjjv5+YmncNWkq7l28tUMGzas2WGqRt5PLaDShQjbkwnIUuKHxx3L6muswaj1R3LT5JvYfZc9mh2SFmHoWkN5/w/3ZuSWo+ardMz53Lac97lzeeiGB3nohgcZs8NYAJZZbhk+8N8fZJX1V2X5lZfn5l//mVvOvLlZ4atGo0aP4oKLz+M9u+86X6XjO0cdwYRtduHyS67g8kuvYK+992xilOqNrxx4MDNmPMNPjj+OAw784hvbv3HINznphJOZeOT3OOmUE5sYoXrD+yk1nnX9pcR2O2zH+qNH+S1qC1n7neswauv150s+AAYNW56N99oEgCfueuKN7f2X6c96W4xk+ZWXrzRO1Wf7HbfjvXvstkA3q+EjhvOpz+4PwE03/LkJkakvpj0yjauvuoZ11l2HLxzw+fn2HTnxCAYNGsT5517AnDlzmhShesP7KS0eJiBSC+o3oPhPt19//xNuZ8sMWAaAAQMsVreKG66fDMDOu0xYIKlcYYUV2HKrLXj55Ze5/dbbmxGeesn7qW5FhY825acXqcXMe30e90/6KwDrvXu9JkejxeX111/nwvMuAmDnXSc0ORrV6sEHHgRg/dHrd7t/VLn9oYceriwm9Z33U1o8/FpNajGTf3EDz057lvW2GMm6JiBta+LhR3H/X//GrrvtwoRdd2p2OKrR7NmzARgyZEi3+4cMHgzArBdeqCwm9Z33UwtwIcKGsAIitZCpv5vClAvvZKV1VuK9R+7e7HC0mJx28i84+fifM2bsaH5x5qnNDkeSpIZquQQkIj4UESdFxI0RMTsiMiLObXZc0uJ218VTue6Eaxm27jA+fMJ/8tbBb212SFoMTj/lDL51yGGM22AsV/zpMoau5BoDrWRwxzfis2Z1u39WxzfqK65YWUzqO++nutWvwkebasUuWEcAGwMvAU8C45objrT4TfntnVx/0nWsPHJl9jn+wwwcOqjZIWkxOOXE0zjs64fztrdvwGVXXsIqq67S7JDUS2PGjgHg4YWMCXik3D56IWMKtGTxfkqLRyvmVl8FxgCDgS8uoq3U8m4/7zauP+k6Vhm9Kvuc8J8mH23q+ONO4LCvH86GG2/IFX+6zOSjRW2/w3YAXH3VNcybN2++fS+++CK33HwrAwcOZPMtNm9GeOol76e6FVHdo021XAKSmddl5kOZmc2ORVrcbjnrZm48bTLDxw4vKh8rDmx2SFoMfnzscUw8/Cg2Gb8xl195CcNWdlXlVjVy1Eh23mUCj01/jNNO+cV8+46eeAxz5sxh349/lEGD/CKhFXg/pcWjFbtg1S0ipixkV9t257rist/z+8v/F4AZTz8NwO233c7nP10UkYatvBLH/uj7TYtPC/rrH+/j5l/9megfrLHRmtz1u6kLtBk8Ygjv2P0db7y+7dzbeP6x5wCY8fCM4jx/+Av/vPdJAFbfaE02et9GFUSvWp1/zgUc+70f0L9/f7bcektO+/npC7RZe521+Nh++zYhOvXFCScfz47b7sQhBx/Kdddez7hxY7nj9ju44frJjB4zmolHf7fZIaoXvJ9S4y2VCcjS6C/3/IXzzzl/vm2PTpvOo9OmA7D2OmubgCxhZv2rGPSYc5Op/9N9zrzmJmvNl4BMv+1Rnrz7ifna/PO+f/LP+/75xmsTkCXLY9MfB2Du3LmcetJp3bbZerutTUBayMhRI7nptps4euLRXDXpKib9cRIjVhvBlw46gMOPPIyhQ51YoJV4P7WA9u0ZVZlo5Z5MEbEDcB1wXmZ+vAHnm7LJphuPv/G2yXXHpiXDxNuOaXYIaqAjN/9Ws0NQAy3bf7lmhyCpG1ttvjV3Tb17ama+s9mxLEkiYgrDlh3P3hWuwXXJo/Dca213L6yASJIkSbWoenB4mw5Eb7lB6JIkSZJalxUQSZIkqVZ+fV8330JJkiRJlbECIkmSJNWqTcdlVKnlEpCIeD/w/vLliPJ5y4g4q/z52cw8tPLAJEmSJC1SyyUgwCbAJ7tsG1k+AB4DTEAkSZKkJVDLjQHJzImZGT081m12jJIkSWpTUeGjTbVcAiJJkiSpdbViFyxJkiSpegH0q3IhwuouVSUrIJIkSZIqYwVEkiRJqpXT8NbNCogkSZKkylgBkSRJkmplAaRuVkAkSZIkVcYERJIkSVJl7IIlSZIk1SgqHISelV2pWlZAJEmSJFXGCogkSZJUi4hqKyBtOuWvFRBJkiRJlbECIkmSJNWoTYsSlbICIkmSJKkyJiCSJEmSKmMXLEmSJKlG/SrsgzW3sitVywqIJEmSpMpYAZEkSZJqEFS7EGHQnosRWgGRJEmSVBkrIJIkSVKNqqyAtCsrIJIkSZIqYwVEkiRJqkVUXAFp02KLFRBJkiRJlTEBkSRJklQZExBJkiSpRhHVPRoTb/woIq6JiCci4pWImBkRd0XEdyNiWJe260ZE9vC4sBExOQZEkiRJal9fBaYCVwEzgEHAFsBE4HMRsUVmPtHlmHuAS7s5132NCMgERJIkSapJVDwNb0OuNTgzX13gzBHfBw4Dvg0c0GX33Zk5sREX745dsCRJkqQ21V3yUfpt+Ty6qlg6WAGRJEmSahBUOw1vAAnjImJKd/sz8511nP595fO93exbPSI+DwwDngNuyczu2vWJCYgkSZLU5iLiUGB5YAjwLmAbiuTjh90036V8dD7+euCTmfl4vbGYgEiSJEk1iupXB/x7nZWODocCwzu9vhLYPzOf6bTtZeBoigHo08ptG1EMWN8RuCYiNsnMOfUE4hgQSZIkqc1l5ojMDGAE8AFgJHBXRIzv1GZGZn4nM6dm5gvlYzKwK3AbsD7wmXpjMQGRJEmSlhKZ+XRmXkKRVAwDzq7hmNeBX5Yvt6s3BrtgSZIkSTWqdhrexSczH4uI+4FNImLlzHx2EYd0dNUaVO+1rYBIkiRJS6fVy+e5NbTdonye1mOrGpiASJIkSbUIiAof9Y53j4gxETGkm+39yoUIVwVuzszny+3jI2KB/CAiJlCsqA5wbn1R2QVLkiRJale7Az+IiJuARynW9BgObE8xCP0p4LOd2v8UGB0RNwNPlts2AnYqfz4yM2+uNygTEEmSJKkGAfSreCHCOl1NMXPVNsCmwIrAHOBB4BzgxMyc2an9OcDewGbAe4FlgKcpVk0/OTNvrD8kExBJkiSpLWXmfcCBvWj/K+BXiy+iggmIJEmSVJOoeBas9phxqysHoUuSJEmqjAmIJEmSpMrYBUuSJEmqUbssRNhMVkAkSZIkVcYKiCRJklQjCyD1swIiSZIkqTJWQCRJkqRaRMVjQNq02mIFRJIkSVJlrIB0EREM6Ofb0i6+vdmhzQ5BkiS1iaDaCkibFkCsgEiSJEmqjgmIJEmSpMrY10iSJEmqkQsR1s8KiCRJkqTKWAGRJEmSahIVV0Das9piBUSSJElSZayASJIkSTVyCEj9rIBIkiRJqowJiCRJkqTK2AVLkiRJqkVUPA1vm3b3sgIiSZIkqTJWQCRJkqQaBNVWQNq0AGIFRJIkSVJ1rIBIkiRJNernPLx1swIiSZIkqTJWQCRJkqQaWQCpnxUQSZIkSZUxAZEkSZJUGbtgSZIkSTWqdCHCNmUFRJIkSVJlrIBIkiRJNYjyf1Verx1ZAZEkSZJUGSsgkiRJUi2i4jEg7VkAsQIiSZIkqTpWQCRJkqQaOQtW/ayASJIkSaqMCYgkSZKkytgFS5IkSaqRPbDqZwVEkiRJUmWsgEiSJEk1chB6/ayASJIkSaqMFRBJkiSpBkG1FZB2rbVYAZEkSZJUGSsgkiRJUi0iqh0D0qbjTayASJIkSaqMCYgkSZKkytgFS5IkSapRm/aKqpQVEEmSJEmVsQIiSZIk1ciFCOtnBUSSJElSZayASJIkSTWyAlI/KyCSJEmSKmMCIkmSJKkydsGSJEmSahBU2wWrXTt7WQFZijz55D/4/Ge+wHprjWLIwKGMHbUBh37t6zz//PPNDk299L3Dj2bv3T7IRqM2Zc0V12H91cay47sn8ONjjmPmczObHZ56YeZzMzn71+fwsX0+waYbvIsRQ9Zg7VXWZbcdd+fsM89l3rx5zQ5RfeDf2/bi/VQri4gfRcQ1EfFERLwSETMj4q6I+G5EDFvIMVtFxB/Ktq9ExL0RcXBE9G9ITJnZiPO0hYiYsun4TcbffPufmx1Kw017ZBo7brsTM2Y8wx577sHYsWO48447ueH6yYwZO4ZrJ1/NsGHd/htsaXNef6nZISwWq62wJhttuiFjx41h5VVX5uU5L3Pn7VO5e8rdjFh9BJNu+ANrrLVGs8NsuAHRfkXbX59+Jl/78qGMWG04226/LWuutQYzZjzDFZf+ntmzZrPn3u/jNxec2ZaDHpftv1yzQ1gslta/t+1qabyfW22+NXdNvXtqZr6z2bEsSSJiynJrDh4/8hvbVHbNaT++iVefnF3XvYiI/wOmAvcDM4BBwBbAu4B/Altk5hOd2u8FXAy8ClwEzATeB4wFfpeZ+/Q1lg7t9//m6tZXDjyYGTOe4SfHH8cBB37xje3fOOSbnHTCyUw88nucdMqJTYxQvfHoMw+z3HILfnj7/neO5Wc/PoHj//tE/vvEHzUhMvXWqNGjuODi83jP7rvSr9+bRenvHHUEE7bZhcsvuYLLL72Cvfbes4lRqjf8e9tevJ9qA4Mz89WuGyPi+8BhwLeBA8ptg4EzgLnADpl5Z7n9SOBa4EMR8ZHMvLCegOyCtRSY9sg0rr7qGtZZdx2+cMDn59t35MQjGDRoEOefewFz5sxpUoTqre6SD4C9PrQXANMenlZlOKrD9jtux3v32G2+5ANg+IjhfOqz+wNw0w3tV5VtV/6L62WLAAAgAElEQVS9bS/eTy0gijEgVT0aMQiku+Sj9NvyeXSnbR8CVgEu7Eg+Op3jiPLlF6mTCchS4IbrJwOw8y4TFviQs8IKK7DlVlvw8ssvc/uttzcjPDXQpP+dBMDbNnxbkyNRIywzYBkABgywWN0q/HvbXryfWkKMi4gp3T3qPO/7yud7O23bqXy+spv2k4GXga0iYtl6Luz/qy0FHnzgQQDWH71+t/tHjV6fq6+6hoceepgdJ+xYZWiq08k/O4U5L81h9uzZ3DPlHm69+TbevuHb+MqhX252aKrT66+/zoXnXQTAzrtOaHI0qpV/b9uL91MLiorH5DXuWhFxKLA8MIRi/Mc2FMnHDzs1G1s+P9j1+Mx8PSIeBd4OjAT+1tdYTECWArNnzwZgyJAh3e4fMngwALNeeKGymNQYpxx/CjOefuaN1xN23YmTzjiBlVdZuYlRqREmHn4U9//1b+y62y5M2HWnRR+gJYJ/b9uL91NLiL83aEKAQ4HhnV5fCeyfmc902tbxj33WQs7RsX3FegKxC5bUwu5/7D6effVp7n/sL/zmojOZ/uhj7LjFztxz172LPlhLrNNO/gUnH/9zxowdzS/OPLXZ4UiS2kBmjsjMAEYAH6CoYtwVEeOrjqWlEpCIGBYRn4mISyLi4XJe4lkRcVNEfDoiWur3qcrgjm9oZnWfzM7q+IZnxbqSWTXRqsNX5T/22p3f/f4inn/ueb706QObHZL66PRTzuBbhxzGuA3GcsWfLmPoSkObHZJ6wb+37cX7qe5UOgh9McjMpzPzEmBXYBhwdqfdHf/Yuy/7vbm9rrJfq31g34diarB3A7cBx1PMU/wO4JfAb6MdJ8uv05ixYwB4+KGHu93/SLl99EL6uKp1rLXOWozdYAx/v/8Bnnv2uWaHo1465cTT+MZXv8Xb3r4BV/zpMoaPGL7og7RE8e9te/F+qp1l5mMUa4O8PSI6+m4/UD6P6do+IgYA6wGvA3VNt9lqCciDwJ7Ampn5scz8dmb+FzAOeAL4IEVJSZ1sv8N2AFx91TULrKr84osvcsvNtzJw4EA232LzZoSnBnvqX08B0L9/QxYrVUWOP+4EDvv64Wy48YZc8afLWGXVVZodkvrAv7ftxfuprgKIqPCx+H+l1cvnueXzteXzbt203Q4YCNycma/Vc9GWSkAy89rMvCIz53XZ/hRwWvlyh8oDW8KNHDWSnXeZwGPTH+O0U34x376jJx7DnDlz2PfjH2XQoEFNilC98fBDjzB71uwFts+bN4/vf+dYnpnxLJtvsRkrDrVLQKv48bHHMfHwo9hk/MZcfuUlDFu5vVZVXpr497a9eD/V6iJiTEQs0J0qIvqVCxGuSpFQPF/u+h3wLPCRiHhXp/bLAceUL+senNhOs2D9u3x+fVENe5g3eVzjwlmynHDy8ey47U4ccvChXHft9YwbN5Y7br+DG66fzOgxo5l49HebHaJqdPWVV3PMkcfy7q02Z+1112allYbyzIxnuPnGW5j+6GOsOmJVfnbqT5odpmp0/jkXcOz3fkD//v3ZcustOe3npy/QZu111uJj++3bhOjUF/69bS/eT3XVYr39dwd+EBE3AY8Cz1HMhLU9xSD0p4DPdjTOzNkR8VmKROT6iLgQmEnRA2lsuf2ieoNqiwSk7JO2X/myu4VTlnojR43kpttu4uiJR3PVpKuY9MdJjFhtBF866AAOP/Iwhg51oGur2H6n7Xj0kencdvNt/OWe+5j1wiwGDhrIqNGj+Pq++/C5L33Ggcst5LHpjwMwd+5cTj3ptG7bbL3d1iYgLcS/t+3F+6kWdzWwPsWaH5tSTJ87h2JYwznAiZk5s/MBmXlpRGwPHE4xvGE54GHga2X7rDeoaMA5mi4ijgMOAf6Qmf9Rx3mmbDp+k/E33/7nxgWnpprz+kvNDkENNCDa4jsTlZbtv1yzQ5DUja0235q7pt49tUFrT7SNiJjy1rWHjB9z2A6VXfPBY6/nlcdntd29aKkxIN2JiIMoko+/A59ocjiSJEmSetDSCUhEHAicQDGF2I5dS0iSJEmSliwt258hIg4GfgbcB0zIzBlNDkmSJEltrsUGoS+RWrICEhHfpEg+7qaofJh8SJIkSS2g5SogEXEkcBQwBdjVbleSJEmqigWQ+rVUAhIRn6RIPuYCNwIHdVMGm56ZZ1UcmiRJkqQatFQCAqxXPvcHDl5ImxuAsyqJRpIkSUuRqHgMSHuWW1pqDEhmTszMWMRjh2bHKUmSJKl7rVYBkSRJkpoiqHYWrPasf7RYBUSSJElSazMBkSRJklQZu2BJkiRJtYiKFyJs0z5YVkAkSZIkVcYKiCRJklQjFyKsnxUQSZIkSZWxAiJJkiTVqNqFCNuTFRBJkiRJlbECIkmSJNXICkj9rIBIkiRJqowJiCRJkqTK2AVLkiRJqkEQlXbBijZdidAKiCRJkqTKWAGRJEmSahEVD0JvzwKIFRBJkiRJ1bECIkmSJNXIWXjrZwVEkiRJUmVMQCRJkiRVxi5YkiRJUo1cCb1+VkAkSZIkVcYKiCRJklQjKyD1swIiSZIkqTJWQCRJkqQaBNVWQNq11mIFRJIkSVJlrIBIkiRJtYiKFyJs0xKIFRBJkiRJlTEBkSRJklQZu2BJkiRJNYmKp+Ftzz5YVkAkSZIkVcYKiCRJklQrFyKsmxUQSZIkSZWxAiJJkiTVqNoxIO3JCogkSZKkylgBkSRJkmoQQL8KCyDtWmuxAiJJkiSpMiYgkiRJkipjFyxJkiSpFlHxIPQ27YNlBUSSJElSZayASJIkSTXq5zS8dbMCIkmSJKkyVkAkSZKkmkTFCxG2Z7XFCogkSZKkylgBkSRJkmoQVPvtfXvWP6yASJIkSaqQCYgkSZKkypiASJIkSTXqF1HZo14RMSwiPhMRl0TEwxHxSkTMioibIuLTEdGvS/t1IyJ7eFxYd1A4BkSSJElqV/sApwL/Aq4DHgeGAx8Afgm8NyL2yczsctw9wKXdnO++RgRlAtLFvExem/tqs8NQgzz/2rPNDkENtOagdZsdgiRpKVftNLx1exDYE/jfzJzXsTEiDgNuBz5IkYxc3OW4uzNz4uIKyi5YkiRJUhvKzGsz84rOyUe5/SngtPLlDlXHZQVEkiRJqkEEDRmb0ZvrAeMiYkp3+zPznXWc/t/l8+vd7Fs9Ij4PDAOeA27JzHvruNZ8TEAkSZKkpUhEDAD2K19e2U2TXcpH52OuBz6ZmY/Xe30TEEmSJKlGTRgD8vc6Kx3d+SHwDuAPmTmp0/aXgaMpBqBPK7dtBEwEdgSuiYhNMnNOPRd3DIgkSZK0lIiIg4BDgL8Dn+i8LzNnZOZ3MnNqZr5QPiYDuwK3AesDn6k3BhMQSZIkaSkQEQcCJwD3Aztm5sxajsvM1ymm7QXYrt447IIlSZIk1ahVv72PiIOBn1Gs5TEhM2f08hTPlM+D6o2lVd9DSZIkSTWIiG9SJB93U1Q+ept8AGxRPk/rsVUNrIBIkiRJNQii2ml4qf9aEXEkcBQwBdi1p25XETGeYhHCeV22TwC+Wr48t96YTEAkSZKkNhQRn6RIPuYCNwIHdTOL1/TMPKv8+afA6Ii4GXiy3LYRsFP585GZeXO9cZmASJIkSTVqwjS89VivfO4PHLyQNjcAZ5U/nwPsDWwGvBdYBnga+C1wcmbe2Iig+pSARERfV0LMzNy4j8dKkiRJqlFmTqRYw6PW9r8CfrW44unQ1wrI6kA2MhBJkiRJ7a9PCUhmrtzoQCRJkqQlXZWD0NuV0/BKkiRJqsxiGYQeEcsAy2fm84vj/JIkSVLVonxUeb121LAKSEQsFxHfi4iHgVd5c7VEImKziPhtRGzUqOtJkiRJaj0NqYBExCDgeuCdwMPAI8CoTk3+BvwHxcqJfZ1BS5IkSWqeqHgMSJuWQBpVATmMIvk4MDPHAOd33pmZL1HMMbxzg64nSZIkqQU1agzIPsC1mXlK+bq7KXqnA+MbdD1JkiSpcs6CVb9GVUDWBqYsos1sYMUGXU+SJElSC2pUAjIHWGURbdYDZjboepIkSZJaUKO6YE0B3hsRAzPz5a47I2IVYDfgqgZdT5IkSapUEESFXbCiTUehN6oCcjIwHLg0ItbuvKN8fQGwPHBSg64nSZIkqQU1pAKSmZdHxHHAocCjFF2yiIjpwFoUk4gdnZk3NOJ6kiRJUjM4CL1+DVuIMDO/AewJXMubC0UOByYDe2Xmdxt1LUmSJEmtqVFjQADIzN8DvweIiLdk5v818vySJElSM1n/qF/DKiBdmXxIkiRJ6qqhFZCIGAF8FNgUGALMAu4CLsjMpxp5LUmSJKlKQbVjQNq12tKwBCQiPg/8FFiO+d+vjwHHRMTXMvMXjbqeJEmSpNbTkAQkIvYGTqWY/eqnwPXAU8AIYEfg88ApEfF0Zl7aiGtKkiRJaj2NqoB8C5gNbJaZD3XZ978RcQZwe9nOBESSJEmtJyqehrdN+2A1ahD6hsBvu0k+AMjMB4DfAhs16HqSJEmSWlCjKiBzgGcX0eZZ4KUGXU+SJEmqXLgQYd0aVQG5BpiwiDYTgKsbdD1JkiRJLahRCcg3gDUj4oyIWLXzjohYNSJ+CawOfLNB15MkSZIqFQT9orpHtOkgkD51wYqIy7vZ/CTwX8DHI+IB4GlgODAWeAtwJ3AysFffQpUkSZLU6vo6BmSPHvYtS/eDzTcDso/XkyRJkpquPWsS1eprArJCQ6OQJEmStFToUwKSmXMaHYgkSZKk9teoaXglSZKktlfpQoRtquEJSESsCKxBMRZkAZk5tdHXlCRJktQaGpaARMQ2wE+Ady2iaf9GXVOSJEmqSlBtBaRday0NWQckIsZTLDI4EjiL4v26FbgAeKx8/Ufgp424niRJkqTW1KiFCA8D5gKbZ+any22TMvPjwBiKxGNr4PQGXU+9MPO5mZz963P42D6fYNMN3sWIIWuw9irrstuOu3P2mecyb968ZoeoXrj4vEsYPeTtPT7GDt2w2WGql5588h98/jNfYL21RjFk4FDGjtqAQ7/2dZ5//vlmh6Y+8H62F++n3hAQEZU92rUE0qguWNsAl2fmo522BUBmvh4RXwd2Bo4GPtKga6pGl158GV/78qGMWG04226/LWuutQYzZjzDFZf+noO+8BWunnQ1v7ngzOIfupZ4G2w4ji9/64Bu99158xRumXwb2+2ybcVRqR7THpnGjtvuxIwZz7DHnnswduwY7rzjTn5+4ilcNelqrp18NcOGDWt2mKqR97O9eD+lxmtUAjIU6Jx8/BsY1PEiMzMibsDkoylGjR7FBRefx3t235V+/d4sen3nqCOYsM0uXH7JFVx+6RXstfeeTYxStXrbRhvwto026HbfPjvvC8BH9v9QlSGpTl858GBmzHiGnxx/HAcc+MU3tn/jkG9y0gknM/HI73HSKSc2MUL1hvezvXg/pcZrVBesZ4EhnV7PANbr5lqDUOW233E73rvHbvMlHwDDRwznU5/dH4CbbvhzEyJTIz3w1we5+457GL76cHZ4z/bNDkc1mvbINK6+6hrWWXcdvnDA5+fbd+TEIxg0aBDnn3sBc+a4/FIr8H62F++nutOvwke7atTv9hDFAPQOdwC7RMQ6ABExDPgA8EiDrqcGWWbAMgAMGOCSMK3uorP+B4B9PvEB+vd3srlWccP1kwHYeZcJC3xJsMIKK7DlVlvw8ssvc/uttzcjPPWS97O9eD+lxaNRCciVwA4R0VEFOQlYAbg7Iq4D/gaMAE5u0PXUAK+//joXnncRADvvOqHJ0ager77yKpdd9Hv69+/Ph/f7YLPDUS88+MCDAKw/ev1u948qtz/00MOVxaS+8362F++nulPpIPQ21agE5HRgD94ceH4d8ElgFrA98Brw9cw8o0HXUwNMPPwo7v/r39h1t12YsOtOzQ5HdfjDJVcye9Zstt15G1Zbc7Vmh6NemD17NgBDhgzpdv+QwYMBmPXCC5XFpL7zfrYX76e0eDSk301mzgSu6bLtXODciOifmXMbcR01zmkn/4KTj/85Y8aO5hdnntrscFSnju5XH/nUPk2ORJKk9hVExQsRtmcVZLGPb2l08hERP4qIayLiiYh4JSJmRsRdEfHdcqyJFuH0U87gW4ccxrgNxnLFny5j6EpDmx2S6vDQ3x5m6m13M2KNEeyw63bNDke9NLjjG9RZs7rdP6vjG9gVV6wsJvWd97O9eD+lxaMVRx5/FZgKXEUx29YgYAtgIvC5iNgiM59oXnhLtlNOPI3Dvn44b3v7Blx25SWssuoqzQ5JdbrQwectbczYMQA8vJA+5I+U20cvpA+6lizez/bi/VR3qqyAtKs+JSARcW8fr5eZuXEfj+0wODNf7Sam71OsyP5toPtV2pZyxx93AhMPP4oNN96QS/9wMcNWtmDU6l579TUuu/By+vfvzz6f+ECzw1EfbL9DUbW6+qprmDdv3nwz7bz44ovccvOtDBw4kM232LxZIaoXvJ/txfspLR597YK1OrBaHx6r1xkv3SUfpd+Wz6PrvUY7+vGxxzHx8KPYZPzGXH7lJSYfbeKPl05i1guz2W4XB5+3qpGjRrLzLhN4bPpjnHbKL+bbd/TEY5gzZw77fvyjDBrkMkqtwPvZXryf0uLRpwpIZq7c6EAa4H3l8yKrMxExZSG7xjUunCXH+edcwLHf+wH9+/dny6235LSfn75Am7XXWYuP7bdvE6JTPToGn//n/g4+b2UnnHw8O267E4ccfCjXXXs948aN5Y7b7+CG6yczesxoJh793WaHqF7wfrYX76e6aufpcavSimNAAIiIQ4HlKVZgfxewDUXy8cNmxrUkemz64wDMnTuXU086rds2W2+3tQlIi3n4gUe485apDj5vAyNHjeSm227i6IlHc9Wkq5j0x0mMWG0EXzroAA4/8jCGDnWiiFbi/Wwv3k+p8SIzmx1Dn0TEU8DwTpuuBPbPzKfrOOeUjTfdePwNt15bd3xaMjzz6lPNDkENtOagdZsdgiS1va0235q7pt49NTPf2exYliQRMWX4mOHj9/v1/pVd8+z/OounH3y67e7FYp+Gd3HJzBGZGRQrrH8AGAncFRHjmxuZJEmSpIVp2S5YHcqKxyURMRV4EDgbeEdzo5IkSVLbiYrHgLTpcJOWrYB0lZmPAfcDb4+IJXGQvCRJkrTUa/kKSBcd0/w2dPV1SZIkCaLihQjbswTSUhWQiBgTEUO62d6vXIhwVeDmzHy++ugkSZIkLUqrVUB2B34QETcBjwLPUcyEtT3FIPSngM82LzxJkiRJPWloAhIR6wMfATYABmXm+8vtawIbATdl5uw6LnE1sD7Fmh+bAisCcygGn58DnJiZM+s4vyRJktStAKLCblHt2QGrgQlIRHwDOKbTOTsvMPJW4ArgQODUvl4jM+8rzyFJkiSpBxExDNgb+A9gQ2AN4P+AvwBnAmdm5rxujtsKOALYguJz/EPAr4GTMrPusdYNGQMSEXtTrEB+M0V14ied92fmQ8BdwF6NuJ4kSZLUDBFR2aMB9gHOAN4N3AYcD1xMsWTFL4HfRpcLRcRewGRgO+AS4GTgLcDPgAsbEVSjKiBfBaYDu2XmqxGxSzdt/krxi0iSJEla/B4E9gT+t3OlIyIOA24HPkixoPfF5fbBFAnLXGCHzLyz3H4kcC3woYj4SGbWlYg0ahasTYA/ZuarPbT5J8WAcUmSJKkl9Yuo7FGvzLw2M6/o2s0qM58CTitf7tBp14eAVYALO5KPsv2rFF2yAL5Yb1yNqoD0p+hP1pOVa2gjSZIk6U3jImJKdzsy8511nPff5fPrnbbtVD5f2U37ycDLwFYRsWxmvtbXCzeqAvIIxSCVbpV9y7YC/tag60mSJEkVC4J+lT0W1zxYETEA2K982TnZGFs+P9j1mMx8nWIZjAEUy1/0WaMqIL8DvhsRX8jM07rZfzAwDjikQdeTJEmSlgZ/r7PS0Z0fUgxE/0NmTuq0vWPB71kLOa5j+4r1XLxRCchPgP8Efh4R+wDLAETERGBbir5ldwOnNOh6kiRJknopIg6iKAr8HfhEM2JoSAKSmXMiYnuKwSx782a96Dvl8yXAZzPTMSCSJElqSQENGRzem+s19HwRBwInAPcDE7pZwLujwjGE7nVsf6GeOBq2EGFmPksxNdcaFONBhlH8Erdm5mONuo4kSZKk3omIgynW8riPIvmY0U2zB4B3AWOA+Qa+l+NG1qMYtD6tnlgaloB0yMx/UM4lLEmSJLWNoFELBNZ8vYacJuKbFOM+7gZ2KQsH3bkW+BiwG3BBl33bAQOByfXMgAWNmwVLkiRJ0hKmXETwhxQVjQk9JB9QTCz1LPCRiHhXp3MsBxxTvjy13pgaUgGJiBNrbJqZ+ZVGXFOSJEmqWiymqXEXh4j4JHAUxcrmNwIHdVPBmZ6ZZwFk5uyI+CxFInJ9RFwIzKRYTX1suf2ieuNqVBesAxexPymKSAmYgEiSJEmL33rlc3+KZTG6cwNwVseLzLy0nFzqcOCDwHLAw8DXgBMzM+sNqlEJyIYL2b4isBnwLeA63izdSJIkSS2l1WbBysyJwMQ+HPdnYPc6L79QjZqG96897P5zRFwO3AP8HuiprSRJkqQ2Vskg9MycBlyGK6FLkiRJS7WGT8Pbg38BH6jwepIkSVIDRbXT8LbQgPfeqKQCEsWd2g54sYrrSZIkSVoyNWoa3vE9nH8t4NMUqyr+phHXkyRJkpqhn8vo1a1RXbDupJhid2GibPP1Bl1PkiRJUgtqVALyU7pPQOYBzwO3A9c1Yt5gSZIkqVmqHQPSnho1De+hjTiPJEmSpPbWkE5sEXFiRHyxEeeSJEmS1L4aNYrm88A6DTqXJEmStMQJii5YlT2a/QsvJo1KQB4HhjXoXJIkSZLaVKMGoV8E7BcRK2Sma31IkiSpDQX9Kq1LtGcNpFEVkGOAB4GrImKHiBjUoPNKkiRJaiONqoDMoEhmBgLXAETEyyw4NW9m5pAGXVOSJEmqTlQ8DW97FkAaloA8SM8LEUqSJElSw9YBeVcjziNJkiQtqQLoV2EFpE0LIH0fAxIR+0XERo0MRpIkSVJ7q2cQ+lnA+xsUhyRJkqSlQKPGgEiSJEltr32XB6xOo6bhlSRJkqRFsgIiSZIk1SToF1V+f9+e1ZZ6E5AVI2Lt3hyQmY/XeU1JkiRJLareBOQr5aNW2YBrSpIkSU1R6UKEbareZGA28EIjApEkSZLU/upNQH6WmUc1JBJJkiRpCecsWPVzFixJkiRJlTEBkSRJklQZB4RLkiRJNQigX4WD0Nu1s5cVEEmSJEmV6XMFJDNNXiRJkrRUcRB6/UwiJEmSJFXGMSCSJElSLaLaMSDtWmyxAiJJkiSpMlZAJEmSpBoEQUR139+363gTE5AuAugfvi3t4vnXZjY7BDXQKsuNaHYIaqBl+y/X7BAkSU1gFyxJkiRJlfGrfkmSJKlG7dotqkpWQCRJkiRVxgqIJEmSVKNKp+FtU1ZAJEmSJFXGCogkSZJUo7ACUjcrIJIkSZIqYwVEkiRJqkEQ9KtwFqx2nXHLCogkSZKkypiASJIkSaqMXbAkSZKkGjkIvX5WQCRJkiRVxgqIJEmSVIuAiAq/v2/TYosVEEmSJEmVsQIiSZIk1SCg4ml425MVEEmSJEmVMQGRJEnS/2/vzuOtqsvFj38eQEFmJRXBAUEGLXPMVBQV0MxrZTezstutbppDpuZQqZmY5dC1HDMtK0tz6JfXMUdUBJxnccAJoTQTFRkEmb+/P9Y6dDicA/sMrHX25vPmtV+L811rr/Wcs14c9rOe7yAVxi5YkiRJUkWi4Gl4a7MTlhUQSZIkSYUxAZEkSZIqFAX+aZN4Iw6MiIsiYkJEzI6IFBFXNXHsgHx/U69r2yImu2BJkiRJtetHwDbAB8AbwLAK3vMMcGMj7c+1RUAmIJIkSVKFih0D0ia+R5Z4vArsAdxXwXueTimNWV0BmYBIkiRJNSqltCzhaC/JkwmIJEmSVIGSFiIcFhFPNLY/pbTDarp0v4g4DOgDvAc8lFJ6tq1ObgIiSZIkqb6989cyETEO+HpK6e+tPbkJiCRJktR+TV6NlY6G5gFnkA1An5K3fRwYA+wF3BMR26aU5rbmIiYgkiRJUkWCiCJXsSh2zEZKaTrw4wbN4yNiH2Ai8EngEOCC1lzHdUAkSZIkNSmltBi4PP9yRGvPZwVEkiRJqlBbLRBYhd7Jt91aeyIrIJIkSZJWZed8O2WlR1XACogkSZJUofaylsbqEBHbky1CuLRB+yiyBQ0BrmrtdUxAJEmSpBoVEQcAB+Rf9s23u0TEFfnf300pnZD//ZfA4Ih4kGz1dMhmwRqZ//3UlNKDrY3JBESSJEmqUBWOAdkW+HqDtoH5C2AaUJeAXAl8HvgE8GlgLeBt4C/AxSmlCW0RkAmIJEmSVKNSSmPI1vGo5NjfAb9bnfGAg9AlSZIkFcgKiCRJklSBiGIHodfqeHcrIJIkSZIKYwVEkiRJqkjQodBB6LVZArECIkmSJKkwVkAkSZKkCtXyQoRFsQIiSZIkqTBWQCRJkqQKhc/vW82foCRJkqTCmIBIkiRJKowJyBrixutv5IRjT2SfvfalX5+N6bF2Lw75+qFlh6WVuOeW+/jfk87j0M98h70GfoqdNtidHx/xk5W+59lHJ3HsV05k9JD92H3TURy8x9e55rK/sGTJkoKiVnPNeG8Gf/r9lXz1i19juy13pG+v/my6/gD23Ws//vSHq1i6dGnZIaoF3njjTQ475HA232QQvbquy9BBW3LCcSfy/vvvlx2aWsD7qTpBNgi9sFfZ3/Bq4hiQNcTPzzqXSc9Oonv37vTr3485L80pOyStwu9/+Sdeef5VunZbhw36bcDcV6at9Pj7b5/AD//nVNbuvDajDxhJr949mHDXg5x36kU88+gkzgiTO1wAACAASURBVP7dGQVFrua48fqbOO67J9B3ow3ZfY/d2XiT/kyf/g633HgrRx9+DGPvHMsfr/mDs65UkSmvTWGv3Ucyffo77P/Z/Rk6dAiPP/Y4v7rwEu6+cyz3jh9Lnz59yg5TFfJ+Sm3PBGQNcfa5Z9Kvf38GbTGQieMnst/e+5cdklbhe2d8lw36rc8mm2/Mkw8+zRGfP7rJYz+YM5czj/85HTp24Nc3XshW2w4D4LAfHsKR/3ks994yjrtuGMs+nx9dVPiq0KDBg7jm+j/zqf32oUOHfxelf/yTHzFqt725+YZbuPnGW/jc5z9bYpRqjmOOOpbp09/hF+efy5FHHbGs/fvH/4CLLriYMaeezkWXXFhihGoO76eWFxRbl6jNh092wVpDjNhzBFsMHuRT1Cqy427bs+nATSq6Z/feMo73353J3geMWpZ8AHTu0pnDTzoEgOuvuHG1xaqW22OvEXx6/32XSz4ANuy7Id889BsATLz/gRIiU0tMeW0KY+++h80GbMbhRx623L5Tx/yIbt26cfVV1zB37tySIlRzeD+l1cMERKoBj094AoBdRn5yhX3b7bINXbp24dnHnmPhgoVFh6ZWWKvTWgB06mSxulrcP248AKP3HrVCUtmjRw922XVn5s2bx6MPP1pGeGom76ca0yGisFetMgGRasC01/4BwKaDNllhX6dOnei36UYsWbyEN6f9s+jQ1EKLFy/m2j9fB8DofUaVHI0q9fJLLwOwxeAtGt0/KG9/5ZVXC4tJLef9lFYPExCpBnww+wMAuvfo1uj+uvY5sz4oLCa1zphTfsILz7/IPvvuzah9RpYdjio0e/ZsAHr16tXo/l49ewIwa+bMwmJSy3k/pdXDur4ktTOXXnwZF5//K4YMHcxlf/h12eFIkuqp3clxi1P1FZCI+K+ISPnrkLLjkcrQvWd3IJsNqzF17T16dS8sJrXMby75LT88/mSGbTmUW+66iXXXW7fskNQMPeueiM+a1ej+WXVP1Hv3LiwmtZz3U1o9qjoBiYhNgIsB+5VojbZZPvbj7/lYkPoWL17MP//+Fh07daT/Zv2KDk3NcMmFl/L97/2QrT66JbfcdRMb9t2w7JDUTEOGDgHg1SbGBLyWtw9uYkyB2hfvpxpyIcK2UbUJSGRzk/4BeA+4tORwpFLtuPsOADx07yMr7HvqoWeYP28+H//Ex1i789pFh6YKnX/uBZx84ilsvc3W3HLXTay/wfplh6QW2GPPEQCMvfueFVaxnzNnDg89+DBdu3Zlp513KiM8NZP3U1o9qjYBAY4GRgLfBJyAW2u0kZ/Zk959enH3jffwwtOTl7UvmL+AS8+6HIAvfOOAssLTKvz8zHMZc8pP2Hb7bbj5jhvo8xFXVa5WAwcNZPTeo5g2dRqXXnLZcvvOGPNT5s6dy8H/9RW6dWt8wgi1L95PrSgIOhT2qtWFCKtyEHpEbAmcDVyQUhofEc2aIiYinmhi17Am2qveLTfdyq03/w2A6W+/DcCjjzzKYd/KVnXt85H1OPOcn5UWn1Y07rbx3H/7BADemz4DgEmPP8/p383uU+/1enPM6d8BslmuTv7F9znpWz/miAOOZu/Pj6Rn755MuPMBpr36d0Z+Zk/2PsCpXNujq6+8hjNPP4uOHTuyy/BduPRXv1nhmE0324Sv/vfBJUSnlrjg4vPZa/eRHH/sCdx37ziGDRvKY48+xv3jxjN4yGDGnHFa2SGqGbyfUturugQkIjoBVwJ/B04uOZyqMemZSVx95dXLtb0+ZSqvT5kKwKabbWoC0s68/Nyr/O26O5Zre3PaP5et5bHRJn2XJSAAe+43gktvvJA/nH8l9916PwsXLGTjzTfm2J8cxZcOPbCiFdVVvGlT/w7AkiVL+PVFjfcmHT5iuAlIFRk4aCATH5nIGWPO4O477+bO2++k70Z9+c7RR3LKqSez7rpOLFBNvJ9aTlDs/6c1+l93pJTKjqFZIuInwCnAbimlh/K2McBpwKEppctbce4ntt1um+0nPDK+TWJV+V6c+WzZIagNDem1VdkhqA117til7BAkNWLXnYbz1JNPP5lS2qHsWNqTiHhii60HbX/xbecVds2j9vser056rebuRVWNAYmIT5JVPX5Rl3xIkiRJqh5V0wUr73r1J+Bl4NSSw5EkSdIaqEOt9osqUDVVQLoDQ4Atgfn1Fh9MZN2vAH6bt51fWpSSJEmSmlQ1FRBgAfC7JvZtD2wHTAReAuyeJUmSpDZVtxBhkderRVWTgKSUPgQOaWxfPgh9O+CPrRmELkmSJGn1qpoERJIkSSpb1GxdojjVNAZEkiRJUpWriQpISmkMMKbkMCRJklTTouCFfWuz2mIFRJIkSVJhTEAkSZIkFaYmumBJkiRJRQif37eaP0FJkiRJhbECIkmSJFUggA4uRNhqVkAkSZIkFcYKiCRJklQhFyJsPSsgkiRJkgpjBUSSJEmqiAsRtgUrIJIkSZIKYwIiSZIkqTB2wZIkSZIq5CD01rMCIkmSJKkwVkAkSZKkSgTFDkKv0WKLFRBJkiRJhbECIkmSJFUggA4FPr+v0QKIFRBJkiRJxbECIkmSJFXEhQjbghUQSZIkSYUxAZEkSZJUGLtgSZIkSRVyIcLWswIiSZIk1aiIODAiLoqICRExOyJSRFy1ivfsGhG3RcSMiPgwIp6NiGMjomNbxGQFRJIkSapQsYPQ28SPgG2AD4A3gGErOzgiPgdcD8wHrgNmAJ8BzgOGA19sbUBWQCRJkqTa9T1gCNATOGJlB0ZET+C3wBJgz5TSt1JKJwLbAg8BB0bEl1sbkAmIJEmSVIEgGwNS3J/WSyndl1J6JaWUKjj8QGB94NqU0uP1zjGfrJICq0hiKmEXLEmSJKn9GhYRTzS2I6W0Qxtfa2S+vaORfeOBecCuEdE5pbSgpRexAiJJkiQJYGi+fbnhjpTSYuB1sgLGwNZcxAqIJEmSVKESpuGdvBoqHU3plW9nNbG/rr13ay5iBUSSJElSYayASJIkSRUJKHQa3sKrLXUVjl5N7K9rn9mai1gBkSRJkgTwUr4d0nBHRHQCNgcWA1NacxETEEmSJKlC1TUJb7Pdm2/3bWTfCKAr8GBrZsACExBJkiRJmb8C7wJfjogd6xojogvw0/zLX7f2Io4BkSRJkioUhY4Bab2IOAA4IP+yb77dJSKuyP/+bkrpBICU0uyIOJQsERkXEdcCM4DPkk3R+1fgutbGZAIiSZIk1a5tga83aBvIv9fymAacULcjpXRjROwBnAJ8AegCvAocB1xY4YrqK2UCIkmSJNWolNIYYEwz3/MAsN/qiAdMQCRJkqSKlTQ4vKY4CF2SJElSYayASJIkSRUIiq2A1GqtxQqIJEmSpMJYAZEkSZIqEVHsNLxVNuVvpayASJIkSSqMFRBJkiSpQs6C1XpWQCRJkiQVxgREkiRJUmHsgiVJkiRVyC5YrWcFRJIkSVJhrIBIkiRJFSp0Gt4aZQKimrbTwV8uOwS1oWk3jCs7BLWhDdbpV3YIkqQSmIBIkiRJFQiKHQNSq7UWx4BIkiRJKowVEEmSJKkiUfAYkNqsgVgBkSRJklQYExBJkiRJhbELliRJklQhFyJsPSsgkiRJkgpjBUSSJEmqkBWQ1rMCIkmSJKkwVkAkSZKkChU7DW9tsgIiSZIkqTBWQCRJkqQKBMWOAanVWosVEEmSJEmFMQGRJEmSVBi7YEmSJEkViYKn4a3NTlhWQCRJkiQVxgqIJEmSVCGn4W09KyCSJEmSCmMFRJIkSaqYFZDWsgIiSZIkqTAmIJIkSZIKYxcsSZIkqRJR8CD0Gu3tZQVEkiRJUmGsgEiSJEkVKnYhwtpkBUSSJElSYayASJIkSRWI/E+R16tFVkAkSZIkFcYKiCRJklShQmfBqlFWQCRJkiQVxgREkiRJUmHsgiVJkiRVqFYHhhfJCogkSZKkwlgBkSRJkipkBaT1rIBIkiRJKowVEEmSJKlCTsPbelZAJEmSJBXGCogkSZJUgcj/FHm9WmQFRJIkSVJhTEAkSZIkFcYuWJIkSVKFHITeelZAJEmSJBXGCogkSZJUoWobGB4RU4HNmtj9dkqpb4HhACYgkiRJUq2bBZzfSPsHRQcCJiCSJElSM1RXBSQ3M6U0puwg6jgGZA1x4/U3csKxJ7LPXvvSr8/G9Fi7F4d8/dCyw1JzvTUPxr6Zvd6cu/y+OQvhtdnw2Dsw/i24502Y8BZMmgGzF5YTr5pl4rgH+fZXjmTHQbswuM9WfGLwcL52wDe5985xZYemZnrjjTc57JDD2XyTQfTqui5DB23JCcedyPvvv192aGoB76fUtqyArCF+fta5THp2Et27d6df/37MeWlO2SGpueYvhpdmQseAJWnF/S/OhNmLoMdasME62XFzFsHbH8L0D2Hr9bJ2tUtn/ugcLrvgcjbq35fR+41ivT7r8t67M5j09HM8POERRn5qz7JDVIWmvDaFvXYfyfTp77D/Z/dn6NAhPP7Y4/zqwku4+86x3Dt+LH369Ck7TFXI+6mGSqh/DIuIJxrbkVLaocJzdI6I/wI2BeYCzwLjU0pL2ijGZjEBWUOcfe6Z9Ovfn0FbDGTi+Inst/f+ZYek5kgJXpgJa3XIkohpjXTZ7NsVPtYFujb4Z/3WPHj+/SxB+UgX6FCVpeOads0fruOyCy7nwIM/z1kX/ZS11157uf2LFi0qKTK1xDFHHcv06e/wi/PP5cijjljW/v3jf8BFF1zMmFNP56JLLiwxQjWH91M1oi9wZYO21yPimyml+4sOxi5Ya4gRe45gi8GDnLu6Wv1jLsxYAFut23QCsWn3FZMPgI26QteOsGgpfOAH2fZmwYIF/O9Pfkn/Tfo1mnwArLXWWiVEppaY8toUxt59D5sN2IzDjzxsuX2njvkR3bp14+qrrmHu3LlNnEHtifdT7cTklNIOjb0qfP8fgFFkSUg3YGvgMmAAcHtEbLNaol4JExCpvZu7CF6dBZt0g3U7t+wcdYmnCWi7M/HeB3jv3Rns+9l96NChA/fccR+//uVl/P6SK3jikafKDk/NdP+48QCM3nsUHTos/19sjx492GXXnZk3bx6PPvxoGeGpmbyfakxEFPZqCyml01NK96aU3k4pzUspPZdSOhz4JbAOMKZNLtQMJiBSe7Y0wXPvQ5dOsEWvlp1j1kKYuxg6d4Du9rpsb555chIAnTt3Zr/hn+N/vvhtzj7tXE7/wc/4z9EHcdC+B/PeO++VHKUq9fJLLwOwxeAtGt0/KG9/5ZVXC4tJLef9VI27NN+OKPrCJiBSe/b6nGwg+Va9s0HlzbVoKTw/I/v7kN5WQNqhuuTisgsuJyL4653X8MJbT3Pnw7cyYtRuPPLAYxz530eXHKUqNXv2bAB69Wr8gUGvnj0BmDVzZmExqeW8n1pRlPBabd7Jt91W50UaYwIitVezFsLUObBZd+jdgq5XS5bCM+/BvCXZOTZ0Bqz2aOnSpQB06tSRy6+7lE/suiPdundj2EeH8purL2Gj/n15eOKjdseSJLW1nfPtlKIvXHUJSERMjYjUxOtfZccntYmlKZu5qmsnGNSz+e9fshSeeg9mLswGpw9uYfctrXY9e2X3d6uPb8Umm2283L51uq7DiFG7A/DME88UHpuar2fdE/FZsxrdP6vuiXrv3oXFpJbzfqqhaqt/RMSWEbFChSMiBgAX519e1crLNFu1dghvV8vJS21uSYJ5i7O/3/vPxo95cWb22qQbDK33n9/ipfB0nnxsZvLR3g0cvDkAPXs3nmj2ytvnf7igsJjUckOGDgHg1SbGBLyWtw9uYkyB2hfvp2rAl4DjI2I8MA2YAwwC/gPoAtwGnFt0UNWagLSr5eSlNtchoF/XxvfNWZS9eq+dVUh61Zu2dfFSeOpdmLUIBvSALVpQPVGhhu+5KxHBq5NfZenSpSvMtPPSi68AsMmAjRt7u9qZPfbMxnKOvfueFe7nnDlzeOjBh+natSs77bxTWSGqGbyfqgH3AUOB7YDhZOM9ZgITydYFuTKl1MjqxqtX1XXBktYIHSNb86Ox10e6ZMds1DX7um+eqCxaCk/mycdAk49qsfGm/Rn96ZG8+Y9/8vtL/rjcvvH3TGD82An07N2TPUYXPkmJWmDgoIGM3nsU06ZO49JLLltu3xljfsrcuXM5+L++QrduhY/5VAt4P9W4aumABSml+1NKX0kpDUsp9U4prZVSWj+ltHdK6U9lJB9QvRWQVi0n39Ry9sCwNoqv3bnlplu59ea/ATD97bcBePSRRznsW9mqrn0+sh5nnvOz0uJTG3j2PZi9CNbpCAl4bfaKx2zQBXqsuNCdynXGL0/j+Wdf4IyTzuTeO8fx0Y9vyT+mvcFdt46lY8eOnHPxz+jZq0fZYapCF1x8PnvtPpLjjz2B++4dx7BhQ3ns0ce4f9x4Bg8ZzJgzTis7RDWD91Nqe9WagLSr5eSrwaRnJnH1lVcv1/b6lKm8PmUqAJtutqkJSLX7cMm/t6/PafyYdTqagLRDG/XfiFvH38AF5/yKsbfdw6MPPEb3Ht0Z/emRHHn8YWy7Y+GL1KoVBg4ayMRHJnLGmDO4+867ufP2O+m7UV++c/SRnHLqyay77rplh6hm8H5qOUGbLRBY6fVqUZRUeWmxiDgNmAA8TzaQZiBwFPBtYD6wS0qpRdPFRMQT2263zfYTHhnfVuGqZD3226rsENSGpt0wruwQ1IY2WKdf2SFIasSuOw3nqSeffjKltEPZsbQnEfHEx7fbevu7H7yjsGvuveu+PPvUpJq7F1VXAUkpnd6g6Tng8Ij4ADiebDn5zxcdlyRJkqRVq6VB6KUtJy9JkiSpMlVXAVmJ0paTlyRJ0pohanVgRoFqqQJS2nLykiRJkipTVQlIe11OXpIkSVJlqq0LVrtcTl6SJElrBrtgtV61JSDtcjl5SZIkSZWpqgQkX2TQhQYlSZKkKlVVY0AkSZIkVbeqqoBIkiRJ5QkiihwDUpvjTayASJIkSSqMCYgkSZKkwpiASJIkSSqMCYgkSZKkwjgIXZIkSapAUOxChLU5BN0KiCRJkqQCWQGRJEmSKlardYniWAGRJEmSVBgrIJIkSVKFrH+0nhUQSZIkSYWxAiJJkiRVKMIaSGtZAZEkSZJUGBMQSZIkSYWxC5YkSZJUMbtgtZYVEEmSJEmFsQIiSZIkVcj6R+tZAZEkSZJUGCsgkiRJUkWCYmsgtVlvsQIiSZIkqTBWQCRJkqQKuRBh61kBkSRJklQYExBJkiRJhTEBkSRJklQYExBJkiRJhXEQuiRJklSBbBLe4gah1+pwdysgkiRJkgpjBUSSJEmqWK3WJYpjBUSSJElSYUxAJEmSJBXGLliSJElSheyA1XpWQCRJkiQVxgqIJEmSVKEIayCtZQVEkiRJUmGsgEiSJEkVCYodBVKb1RYrIJIkSZIKYwVEkiRJqlBt1iSKZQVEkiRJUmFMQCRJkiQVxi5YkiRJUsXshNVaVkAkSZIkFcYKiCRJklSJKHghwhottlgBkSRJkmpYRGwcEb+PiH9GxIKImBoR50fEumXEYwVEkiRJqlERMQh4ENgAuAmYDOwEHAPsGxHDU0rvFRmTFRBJkiSpdl1ClnwcnVI6IKX0w5TSSOA8YCjws6IDMgGRJEmSKhBAFPqnlfFm1Y99gKnArxrsPg2YC3wtIrq18lLNYhes5Q14afLL7P7JEWXHobby6vSyI1Ab+o/dDyg7BLWhtTqsVXYIkhox+cWXAAaUHEa7NPnFl9h1p+GFXg8YFhFPNLY/pbTDKk6xV769K6W0tMF750TEA2QJys7APa0Mt2ImIMub/eGHH/L0U89MLTuQ1WxYvp1cahRqK2vM/Xzu6efLDqEIa8z9XEN4P2vLmnI/BwCzyw6iHZr84Ycf8tSTTxd93QGteO/QfPtyE/tfIUtAhmACUo6U0uZlx1CEuiy6gqxZVcD7WVu8n7XF+1lbvJ9rtpTSV8uOoQV65dtZTeyva+9dQCzLOAZEkiRJUmFMQCRJkqTaVFfh6NXE/rr2mQXEsowJiCRJklSbXsq3Q5rYPzjfNjVGZLUwAZEkSZJq0335dp+IWO5zf0T0AIYD84CHiwzKBESSJEmqQSml14C7yGbS+k6D3acD3YArU0pzi4wrUkpFXk+SJElSQfLFCB8kWw39JuBF4JNka4S8DOyaUnqv0JhMQCRJkqTaFRGbAD8B9gX6AG8BNwCnp5TeLzweExBJkiRJRXEMiCRJkqTCmIBIkiRJKowJiCRJkqTCmIBIkiRJKowJiCRJkqTCmIBIkiRJKowJiCRJkqTCmIBIkiRJKowJiFSlIqJj2TGo7URElB2D2ka+4rBqjP9GpbZjAiJJ7cO6ZQeg1ouI64GzI+KjZceithER10bE8JRSKjsWqVZ0KjsAFSciOqSUlpYdh1onIr4AbAN8IiLGAg+klB4uOSy1UER8G9gD+HRETABuTin9ruSw1AIRcRLweWApMD8izk0pvVhyWGqFiPgb8Gngvoh4yP9DpbZhArIGiIivAreklGabhFS3iLgK+BJQ1/3qU8DjEXF6Sulv5UWmlsjv5wHALOANsg86u0bE9JTSLaUGp2bJu+d0ApYAfwO+kTf/r0lIdYqI24E9geOAa/2/U2o7dsGqcRFxI/An4AcR0T2ltDQivO9VKCL+SvZ09QpgR+Ag4I/A9sCBEdHJPsrVI++q8zng18AngOHAMUAf4GMlhqYWyLvnTAQWALcBt5MlISdGxJYlhqYWyJOPvYCTgStSSrNKDkmqKVZAalhEnA18FvgQ+C6QIuLslNIHVkKqS0T8iOxJ3FnAJSmlGcCTEfEK2QfXg4BzUkqTy4tSlYqIs4CRwM/J7ud7efs4YCGOB6lWs4EAJgEP5W3fAIiI81JKkyKiN9AlpfSvckLUquQP7vYAjgf+klKaGRGdgLWBg4H1yaqWr6WU7iwvUql6+SS8RkXEAcC3gcnA4cC/yMrIP7QSUl0iYivgv4EXgN+llGbUzYCVUnoGuAtYB+hVXpSqVETsDRxC9oT8srrkIzec7MHQ0oi4JiJujoifWtmqGs+T/a4dnVKaBJzDv7tjHR4RI4EngbMioltpUapJEfELsgd3T5BVPt6JiHWBLwD3Ab8BfgZcDNwWEeeWFqxUxayA1KCI6EI2TmAd4JsppUci4h3gV2RPdLASUlUGAP2B41JKb0VEpJSWRESnlNJi4B/5cf1Ki1DN8RowH/h5Sml6XWNEjACOInsw9AlgLWBrYH9gC+DLxYeqZloIvE12/0gpTchzxwXAEcDXyP7fnZhSmltWkFqp35J1vRoO/CwijiMba3cp8Bbw03y7Gdn/p8dFxJKU0g9KileqSiYgNSilND8ifg48mFJ6JG8eS1YJuZRGkpDsbU4x2E49AhwNPAzL+ppDNtgV4M18Ox+c7ay9SylNiYihKaV5EdExTyY/CZwNDCPr4nFH3u1ja+Bq4KCIeDSl9MsyY1fT6v7dRcRjwMiIWAdYmCchfYD9gO5kXbOcta6dSilNjoiDgP9H9nt3E7Jxds8Dn0opfVB3bETcC9wKHBYRt6aUJpQRs1SN7IJTo1JKT5ElG3VfLwbuJUtC3iRLQpZ1x6r7UBsR65cRr5qWd9G5KqX0boP2hgnjh3n7UoCI2CMivlJMlGqmuntVl0QuJqty/EdK6dqU0sx8/yTgJLJpXTctI1BVpl7S/ywwGNg8Ty43Ai4CFgHPAbsAR0XENuVEqlVJKb0KfBF4hmzij3eB/fMHdp1gWcJ5F9lEID2BDcuKV6pGJiA1LKW0qMHXS8iSkCOAf5IlISfX+4V6CHB7/jRW7UhKaUHDtnrjAuo+xK5Tb9+ngPOBMfnTV7UjDZPHlNITwNCU0j11Y7MiYq189wyy39Vdio1SLTSNbJrsRfm/vceBrmS/b48A/g84DDgkItYuLUqtVJ6EHATcTzZRxPt599fFDQ59O9/2LTRAqcrZBWsNkz+Rq6uEXAocCyyJiDeBU4D1gJklhqjKBZD495ogC2BZ8nEW2RP14Q0GOaudyT/UJPJ/d3k3nqj3AOFAsu51tzU4Xu3To/x70o/9yJKPU1JKl8OyxHIR2YfahaVFqVVKKb0SEV+u93VdT4GoV/HaEnifbApmSRUyAVkD5UnI/cC3gMuB75E9PZ8NfDKl9FKZ8aky9f4D7Jxv5+cDmc8GBgG75l141I7Vfaip9+Fm2RiefDa7g8g+1DYcA6T2ax5ZleN1si50l9XtSCmNi4iHU0rzywpOlUspvV3/6wb/Pr9INj36A8DUwoOTqphdsNZA+S/QhWSl5f8je0I3E9gtpfRcqcGpJeq6Yu0H/IIs+djN5KP61H+yGhFfA84g63p1eP0Zs9R+pZRmA98km8b1UrKpllNEdKjrNmnyUZ0aJB9fAX5MNn7rhLpxW5IqYwVkDVTvyfk3gK+SLai0e0rphdKCUrPV64pTN1bgu2QPFYanlJ4tLzK1VP5BtRtwHjA6b97TBSarS0rpwYjYH5hel3w4M131y7tHdiFb3+U/yBYmHJVSerncyKTqYwVkDRURuwFjyMZ87GbyUX3qdcWZlW87ADubfFSviOhK1iXyi8DTwD5WJatTSultk4/akj8cOBU4FJhCtuCk/z6lFgi7E6+ZImJDsgTk4pTS8yWHo1aIiL5kXQEucPxO9ctXXd4CeMVuHVL7EhEbADsAjzWcGl1S5UxA1mD1VtJWlfNeSlIxnIlOaj0TEEmSJEmFcQyIJEmSpMKYgEiSJEkqjAmIJEmSpMKYgEiSJEkqjAmIJEmSpMKYgEiSJEkqjAmIJEmSpMKYgEiSJEkqjAmIJOUiYkBEpIi4okH7FXn7gFICa6bmxhsR4yKi1avSRsTUiJja2vOs4hptEqskqTwmIJIKlX8wrv9aEhHvRsS9EXFw2fGtDk0lNpIkrYk6fLq3awAAB8lJREFUlR2ApDXW6fl2LWAY8Dlgr4jYMaV0XHlhNeok4GzgzbIDkSSp2pmASCpFSmlM/a8jYhRwN3BsRFyYUppaRlyNSSm9BbxVdhySJNUCu2BJahdSSvcAk4EAPgHLd12KiCERcV1ETI+IpRGxZ917I2K9iDgrIl6MiA8jYlZE3BMR+zR2rYjoERG/jIg3ImJ+REyOiONo4nfiysZURMROeVxvRsSCiHgrIu6KiIPy/WOA1/PDv96g+9k3GpzrUxFxW94lbUFEvBYR/xsRvZuIa3RETIiIuRExIyJujIhhK/kxVywi1o6Io/J4puXxzIiIsRHx6VW8t1dEXJz/TOZHxAsRcXRERBPHfzIi/hoR/4qIhRHxj4i4LCL6tcX3IklqX6yASGpP6j6gNhxkPAh4BHgZ+DOwDjAbICI2A8YBA4AJwB1AN2B/4I6IOCyl9NtlF4joDNxDluQ8k5+vN3AqsEezgo04FPg1sAS4GXgF2ADYETgS+EseW2/gmPx6N9Y7xdP1znUaMAaYAdwKTAc+DpwA7BcRu6SUZtc7/kDgOmBhvn0L2A14CHi2Od9HE9YDLgAeJKtMvQNsBHwGuC0iDk0pXd7I+9YGxpJ9z9fmX38hP9dQ4Dv1D46I/wF+Aywg+xn+AxgMHAJ8JiJ2Tin9vQ2+H0lSe5FS8uXLl6/CXmTJRWqkfTSwNH9tlrcNqDseOLOJ843L3/PlBu29yT7gfwhsWK/95Px81wMd6rVvTvbhPwFXNDjXFXn7gHptWwGL8vd8tJG4Nq739wGNnbfe/r3y/Q8CvRvs+0a+77x6bd2B9/Lr79jg+PPq/cwGNHa9Jn6GqUFb5/rfQ732XsBz+fe9ToN9U/PrTgQ612tfD3gt3zeiXvsQsgTqVaB/g3ONIkvsblhVrL58+fLlq7pedsGSVIqIGJO/fhYRfyWrXARwfkppWoPD3+bfg9brn2MbsqrF9Smla+vvSynNBE4DupA9ga/zTbKE5fsppaX1jn8duLAZ38IRZFXkM1JKzzfcmVJ6oxnnOjrfHprHXf88V5AlUl+t1/w5sg/1V6eUHm9wrjHArGZcu1EppQWNfQ8ppVnA74F1ybvKNeKklNKCeu+ZAZyRf/nNescdQTYJwTEppeUG+KesS97NZFWQHi3+RiRJ7Y5dsCSV5bR8m4CZZN2nfpdSuqqRY5+p/4G2nl3yba98rEVD6+fbLSEb+wFsAfwjpfRaI8ePqxfXquycb2+v8PiV2YWsmvHFiPhiI/vXBtaPiD4ppfeA7fP2+xsemFKaFRFP08zuZI2JiI8CJwIjyLpfdWlwSP9G3raYrJLT0Lh8u129trr7t0dENJbMbAB0JKuUPFFZ1JKk9s4ERFIpUkqNDkhuwr+aaO+Tb/fOX03pnm975du3m3mdxtQNDG+LqXn7kP0+XlXyU9f1qi2/j0ZFxM7AvXlcddWI2WTVo23JqjCdG3nruymlJSuJqVe9trr7d+Iqwum+iv2SpCpiAiKpGjS18nVdV6NjUkqVdJ+qO37DJvb3bUZMdV2l+pPN3tUas8jGo6zXjOOhbb6PpvyIbLD/XimlcfV3RMRJZAlIYz4SER0bSULqYqrfPazu771SvQH2kqTa5hgQSdXs4Xy7eyUHp5TmkA94johBjRyyZwuuvdIpaXN1H8Y7ruRc6+ZdnirxZL5doZtVRPQiq1C01hbAjIbJR1PXracTsGsj7Xvm26fqtTXr/kmSaoMJiKSqlQ/AngD8Zz6d6woiYuuI2KBe0x/IfvedExEd6h23Of8eDF6JX5ONdzg1IrZq5Lob1/vyfbIqzqZNnOu8fPvbxta+iIhueZeoOjfl5zw4InZscPgYlu/m1FJTgfUi4uMNYvkW8KlVvPesfLrjuvesR1ZRgeznX+disrEv50XEkIYnydciMTmRpBpjFyxJ1e5gsrEKv4uIo8nWC5kJbEy2jsbHyAY7T8+P/wVwANnMWE9GxJ1k4zkOAsYDn63koimlFyLiSOBS4KmIuIlsHZA+ZLNDzSabXpeU0gcR8Qiwe0T8mWw9kyXAzSmlZ1NK90TED4GzgFci4jayxQu7A5uRVRwmAvvWO9+3ydb/mBAR9dcB+Vj+fYxo1k9xReeTJRoTI+IvZN2ldsyv8VfgwCbe9xbZ2JDnIuJmslmuDiQbxH5JSml83YEppcl54vh74PmIuCP/2axFlqztTrb+SJssrihJah9MQCRVtZTSGxGxA/BdsqTiq2Rdnf4FvABcBEyqd/yCiBhNVin4EtkCgVOBnwI3UGECkp/rtxHxHNligXuSJTbvki0E2HCRvq+RVTr2Bb5CNuXwG/mxpJTOiYgHyKowu5GNsZhFNsj9N8DVDa7914jYl2zg+kFkC/mNJ0u2fkgrE5CU0h0R8RmyysWXyBKmR8mSqoE0nYAsJFvT5Uzgy8BHgCnA2WT3ouF1roqIZ4Dj83PvA8wF/kmW6FzXmu9DktT+REpNje2UJEmSpLblGBBJkiRJhTEBkSRJklQYExBJkiRJhTEBkSRJklQYExBJkiRJhTEBkSRJklQYExBJkiRJhTEBkSRJklQYExBJkiRJhTEBkSRJklQYExBJkiRJhTEBkSRJklQYExBJkiRJhTEBkSRJklQYExBJkiRJhTEBkSRJklQYExBJkiRJhfn/TqRhz1yiwjIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x360 with 2 Axes>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 351,
       "width": 400
      },
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import itertools\n",
    "import numpy as np\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format='retina'\n",
    "\n",
    "cm = confusion_matrix(y_true=y_true, y_pred=y_pred)\n",
    "\n",
    "plt.figure()\n",
    "fig, ax = plt.subplots(figsize=(10,5))\n",
    "plot_conf_mat(cm, \n",
    "              classes=['1', '2', '3', '4', '5'], \n",
    "              title='Confusion Matrix')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "Jupyter.notebook.save_checkpoint();\n",
       "Jupyter.notebook.session.delete();\n"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%javascript\n",
    "Jupyter.notebook.save_checkpoint();\n",
    "Jupyter.notebook.session.delete();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "instance_type": "ml.t3.medium",
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
